将chunk回收到chunkpool中

Change-Id: I752418f4ba3976d082a68eb645f0d1eb1e71283c

Author: lixiaocui1

conf/chunkserver.conf.example

@@ -86,3 +86,9 @@ chunkfilepool.chunk_file_pool_dir=./0/
 #chunkfilepool.meta_path=./chunkfilepool.meta
 chunkfilepool.cpmeta_file_size=4096
 chunkfilepool.retry_times=5
+
+#
+# trash settings
+#
+trash.expire_afterSec=86400
+trash.scan_periodSec=3600

deploy/local/chunkserver/conf/chunkserver.conf.0

@@ -86,3 +86,9 @@ chunkfilepool.chunk_file_pool_dir=./0/chunkfilepool/
 chunkfilepool.meta_path=./0/chunkfilepool.meta
 chunkfilepool.cpmeta_file_size=4096
 chunkfilepool.retry_times=5
+
+#
+# trash settings
+#
+trash.expire_afterSec=120
+trash.scan_periodSec=60

deploy/local/chunkserver/conf/chunkserver.conf.1

@@ -86,3 +86,9 @@ chunkfilepool.chunk_file_pool_dir=./1/chunkfilepool/
 chunkfilepool.meta_path=./1/chunkfilepool.meta
 chunkfilepool.cpmeta_file_size=4096
 chunkfilepool.retry_times=5
+
+#
+# trash settings
+#
+trash.expire_afterSec=120
+trash.scan_periodSec=60

deploy/local/chunkserver/conf/chunkserver.conf.2

@@ -87,3 +87,8 @@ chunkfilepool.meta_path=./2/chunkfilepool.meta
 chunkfilepool.cpmeta_file_size=4096
 chunkfilepool.retry_times=5
 
+#
+# trash settings
+#
+trash.expire_afterSec=120
+trash.scan_periodSec=60

src/chunkserver/chunkserver.h

@@ -9,11 +9,13 @@
 #define SRC_CHUNKSERVER_CHUNKSERVER_H_
 
 #include <string>
+#include <memory>
 #include "src/common/configuration.h"
 #include "src/chunkserver/copyset_node_manager.h"
 #include "src/chunkserver/heartbeat.h"
 #include "src/chunkserver/clone_manager.h"
 #include "src/chunkserver/register.h"
+#include "src/chunkserver/trash.h"
 
 namespace curve {
 namespace chunkserver {
@@ -53,6 +55,9 @@ class ChunkServer {
     void InitRegisterOptions(common::Configuration *conf,
         RegisterOptions *registerOptions);
 
+    void InitTrashOptions(common::Configuration *conf,
+        TrashOptions *trashOptions);
+
     int GetChunkServerMetaFromLocal(const std::string &storeUri,
         const std::string &metaUri,
         const std::shared_ptr<LocalFileSystem> &fs,
@@ -65,10 +70,17 @@ class ChunkServer {
     // false-停止运行
     volatile bool toStop_;
 
-    // chunkserver包含的模块
+    // copysetNodeManager_ 管理chunkserver上所有copysetNode
     CopysetNodeManager copysetNodeManager_;
+
+    // cloneManager_ 管理克隆任务
     CloneManager cloneManager_;
+
+    // heartbeat_ 负责向mds定期发送心跳，并下发心跳中任务
     Heartbeat heartbeat_;
+
+    // trash_ 定期回收垃圾站中的物理空间
+    std::shared_ptr<Trash> trash_;
 };
 
 }  // namespace chunkserver

src/chunkserver/config_info.h

@@ -11,6 +11,7 @@
 #include <string>
 
 #include "src/fs/local_filesystem.h"
+#include "src/chunkserver/trash.h"
 
 namespace curve {
 namespace chunkserver {
@@ -75,6 +76,10 @@ struct CopysetNodeOptions {
     std::shared_ptr<ChunkfilePool> chunkfilePool;
     // 文件系统适配层
     std::shared_ptr<LocalFileSystem> localFileSystem;
+    // 回收站, 心跳模块判断该chunkserver不在copyset配置组时，
+    // 通知copysetManager将copyset目录移动至回收站
+    // 一段时间后实际回收物理空间
+    std::shared_ptr<Trash> trash;
 
     CopysetNodeOptions();
 };

src/chunkserver/copyset_node.h

@@ -127,6 +127,12 @@ class CopysetNode : public braft::StateMachine,
      */
     CopysetID GetCopysetId() const;
 
+    /**
+     * 返回复制组数据目录
+     * @return
+     */
+    std::string GetCopysetDir() const;
+
     /**
      * 返回当前副本是否在leader任期
      * @return
@@ -225,12 +231,6 @@ class CopysetNode : public braft::StateMachine,
      */
     virtual void Propose(const braft::Task &task);
 
-    /**
-     * 删除复制组持久化数据
-     * @return
-     */
-    int RemoveCopysetData();
-
     /**
      * 获取复制组成员
      * @param peers:返回的成员列表(输出参数)
@@ -386,6 +386,8 @@ class CopysetNode : public braft::StateMachine,
     std::string chunkDataApath_;
     // chunk file的相对目录
     std::string chunkDataRpath_;
+    // copyset绝对路径
+    std::string copysetDirPath_;
     // 文件系统适配器
     std::shared_ptr<LocalFileSystem> fs_;
     // Chunk持久化操作接口

src/chunkserver/copyset_node_manager.cpp

@@ -244,23 +244,24 @@ bool CopysetNodeManager::DeleteCopysetNode(const LogicPoolID &logicPoolId,
 
 bool CopysetNodeManager::PurgeCopysetNodeData(const LogicPoolID &logicPoolId,
                                               const CopysetID &copysetId) {
-    bool ret = false;
     /* 加写锁 */
     WriteLockGuard writeLockGuard(rwLock_);
     GroupId groupId = ToGroupId(logicPoolId, copysetId);
     auto it = copysetNodeMap_.find(groupId);
     if (copysetNodeMap_.end() != it) {
         it->second->Fini();
-        if (0 != it->second->RemoveCopysetData()) {
+        if (0 != copysetNodeOptions_.trash->RecycleCopySet(
+                it->second->GetCopysetDir())) {
             LOG(ERROR) << "Failed to remove copyset <" << logicPoolId
                        << ", " << copysetId << "> persistently";
-            ret = false;
+            return false;
         }
         copysetNodeMap_.erase(it);
-        ret = true;
+        LOG(INFO) << "Success to remove copyset <" << logicPoolId
+                   << ", " << copysetId << "> to trash";
     }
 
-    return ret;
+    return true;
 }
 
 bool CopysetNodeManager::IsExist(const LogicPoolID &logicPoolId,