Merge "add snapshot throttle" into feature

charlie0129 · Jun 22, 2019 · 9e30c51 · 9e30c51
2 parents 57f7788 + d2e43ea
commit 9e30c51
Show file tree

Hide file tree

Showing 12 changed files with 73 additions and 6 deletions.
diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example
@@ -4,25 +4,41 @@
 #global.logfile=/root/data/log/0/chunkserver.log
 global.ip=127.0.0.1
 global.port=8200
+# chunk大小，一般16MB
 global.chunk_size=16777216
+# chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 
 #
 # MDS settings
 #
 mds.ip=127.0.0.1
 mds.port=6666
+# 向mds注册的最大重试次数
 mds.register_retries=100
+# 向mds注册的rpc超时时间，一般1000ms
 mds.register_timeout=1000
-mds.heartbeat_interval=1
+# 向mds发送心跳的间隔，一般10s
+mds.heartbeat_interval=10
+# 向mds发送心跳的rpc超时间，一般1000ms
 mds.heartbeat_timeout=5000
 
 #
 # Chunkserver settings
 #
+# chunkserver主目录
 chunkserver.stor_uri=local://./0/
+# chunkserver元数据文件
 chunkserver.meta_uri=local://./0/chunkserver.dat
+# disk类型
 chunkserver.disk_type=nvme
+# raft内部install snapshot带宽上限，一般80MB
+chunkserver.snapshot_throttle_throughput_bytes=41943040
+# check cycles是为了更精细的进行带宽控制，以snapshotThroughputBytes=100MB，
+# check cycles=10为例，它可以保证每1/10秒的带宽是10MB，且不累积，例如第1个
+# 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
+# 不是20MB的带宽
+chunkserver.snapshot_throttle_check_cycles=4
 
 #
 # Testing purpose settings
@@ -35,33 +51,51 @@ test.testcopyset_conf=127.0.0.1:8200:0,127.0.0.1:8201:0,127.0.0.1:8202:0
 #
 # Copyset settings
 #
+# 是否检查任期，一般检查
 copyset.check_term=true
+# 是否关闭raft配置变更的服务，一般不关闭
 copyset.disable_cli=false
 copyset.log_applied_task=false
+# raft选举超时时间，一般是5000ms
 copyset.election_timeout_ms=5000
+# raft打快照间隔，一般是600s
 copyset.snapshot_interval_s=600
+# 当落后节点落后日志条目超过catchup_margin，会通过install snapshot恢复
 copyset.catchup_margin=20000
+# copyset chunk数据目录
 copyset.chunk_data_uri=local://./0/copysets
+# raft wal log目录
 copyset.raft_log_uri=local://./0/copysets
+# raft元数据目录
 copyset.raft_meta_uri=local://./0/copysets
+# raft snapshot目录
 copyset.raft_snapshot_uri=local://./0/copysets
+# copyset回收目录
 copyset.recycler_uri=local://./0/recycler
 copyset.max_inflight_requests=5000
 
 #
 # Clone settings
 #
+# 克隆的分片大小，一般1MB
 clone.slice_size=1048576
+# 克隆的线程数量
 clone.thread_num=10
+# 克隆的队列深度
 clone.queue_depth=100
+# curve用户名
 curve.root_username=root
+# curve密码
 curve.root_password=
+# client配置文件
 curve.config_path=conf/client.conf
+# s3配置文件
 s3.config_path=conf/s3.conf
 
 #
 # Local FileSystem settings
 #
+# 是否开启使用renameat2，ext4内核3.15以后开始支持
 fs.enable_renameat2=true
 
 #
@@ -76,20 +110,29 @@ storeng.sync_write=false
 #
 # Concurrent apply module
 #
+# 并发模块的并发度，一般是10
 concurrentapply.size=10
+# 并发模块线程的队列深度
 concurrentapply.queuedepth=1
 
 #
 # Chunkfile pool
 #
+# 是否开启从chunkfilepool获取chunk，一般是true
 chunkfilepool.enable_get_chunk_from_pool=false
+# chunkfilepool目录
 chunkfilepool.chunk_file_pool_dir=./0/
+# chunkfilepool meta文件路径
 #chunkfilepool.meta_path=./chunkfilepool.meta
+# chunkfilepool meta文件大小
 chunkfilepool.cpmeta_file_size=4096
+# chunkfilepool get chunk最大重试次数
 chunkfilepool.retry_times=5
 
 #
 # trash settings
 #
+# chunkserver回收数据彻底删除的过期时间
 trash.expire_afterSec=86400
+# chunkserver检查回收数据过期时间的周期
 trash.scan_periodSec=3600
diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.0 b/deploy/local/chunkserver/conf/chunkserver.conf.0
@@ -23,6 +23,8 @@ mds.heartbeat_timeout=5000
 chunkserver.stor_uri=local://./0/
 chunkserver.meta_uri=local://./0/chunkserver.dat
 chunkserver.disk_type=nvme
+chunkserver.snapshot_throttle_throughput_bytes=41943040
+chunkserver.snapshot_throttle_check_cycles=4
 
 #
 # Testing purpose settings

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.1 b/deploy/local/chunkserver/conf/chunkserver.conf.1
@@ -23,6 +23,8 @@ mds.heartbeat_timeout=5000
 chunkserver.stor_uri=local://./1/
 chunkserver.meta_uri=local://./1/chunkserver.dat
 chunkserver.disk_type=nvme
+chunkserver.snapshot_throttle_throughput_bytes=41943040
+chunkserver.snapshot_throttle_check_cycles=4
 
 #
 # Testing purpose settings

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.2 b/deploy/local/chunkserver/conf/chunkserver.conf.2
@@ -23,6 +23,8 @@ mds.heartbeat_timeout=5000
 chunkserver.stor_uri=local://./2/
 chunkserver.meta_uri=local://./2/chunkserver.dat
 chunkserver.disk_type=nvme
+chunkserver.snapshot_throttle_throughput_bytes=41943040
+chunkserver.snapshot_throttle_check_cycles=4
 
 #
 # Testing purpose settings

diff --git a/deploy/local/chunkserver/start_chunkservers_locally.sh b/deploy/local/chunkserver/start_chunkservers_locally.sh
@@ -17,6 +17,6 @@ ${curveformt}/curve_format -chunkfilepool_dir=./0/chunkfilepool/ -chunkfilepool_
 ${curveformt}/curve_format -chunkfilepool_dir=./1/chunkfilepool/ -chunkfilepool_metapath=./1/chunkfilepool.meta -filesystem_path=./1/ -allocateByPercent=false -preallocateNum=16
 ${curveformt}/curve_format -chunkfilepool_dir=./2/chunkfilepool/ -chunkfilepool_metapath=./2/chunkfilepool.meta -filesystem_path=./2/ -allocateByPercent=false -preallocateNum=16
 
-${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=1 -v 19 -conf=${conf}/chunkserver.conf.0 2>${loghome}/0/chunkserver.err &
-${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=1 -v 19 -conf=${conf}/chunkserver.conf.1 2>${loghome}/1/chunkserver.err &
-${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=1 -v 19 -conf=${conf}/chunkserver.conf.2 2>${loghome}/2/chunkserver.err &
+${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_max_install_snapshot_tasks_num=5 -raft_sync=true -minloglevel=1 -v 19 -conf=${conf}/chunkserver.conf.0 2>${loghome}/0/chunkserver.err &
+${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_max_install_snapshot_tasks_num=5 -raft_sync=true -minloglevel=1 -v 19 -conf=${conf}/chunkserver.conf.1 2>${loghome}/1/chunkserver.err &
+${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_max_install_snapshot_tasks_num=5 -raft_sync=true -minloglevel=1 -v 19 -conf=${conf}/chunkserver.conf.2 2>${loghome}/2/chunkserver.err &
diff --git a/include/chunkserver/chunkserver_common.h b/include/chunkserver/chunkserver_common.h
@@ -9,6 +9,8 @@
 #define INCLUDE_CHUNKSERVER_CHUNKSERVER_COMMON_H_
 
 #include <braft/configuration.h>
+#include <braft/raft.h>
+#include <braft/snapshot_throttle.h>
 #include <braft/file_system_adaptor.h>
 
 #include <cstdint>
@@ -39,6 +41,8 @@ using NodeOptions = braft::NodeOptions;
 using FileSystemAdaptor = braft::FileSystemAdaptor;
 using DirReader = braft::DirReader;
 using PosixFileSystemAdaptor = braft::PosixFileSystemAdaptor;
+using SnapshotThrottle = braft::SnapshotThrottle;
+using ThroughputSnapshotThrottle = braft::ThroughputSnapshotThrottle;
 
 
 // TODO(lixiaocui): 考虑一下后续的单元测试或者校验要怎么做

diff --git a/src/chunkserver/chunkserver.h b/src/chunkserver/chunkserver.h
@@ -83,6 +83,9 @@ class ChunkServer {
 
     // trash_ 定期回收垃圾站中的物理空间
     std::shared_ptr<Trash> trash_;
+
+    // install snapshot流控
+    scoped_refptr<SnapshotThrottle> snapshotThrottle_;
 };
 
 }  // namespace chunkserver

diff --git a/src/chunkserver/config_info.cpp b/src/chunkserver/config_info.cpp
@@ -30,7 +30,8 @@ CopysetNodeOptions::CopysetNodeOptions()
       pageSize(4096),
       concurrentapply(nullptr),
       chunkfilePool(nullptr),
-      localFileSystem(nullptr) {
+      localFileSystem(nullptr),
+      snapshotThrottle(nullptr) {
 }
 
 }  // namespace chunkserver

diff --git a/src/chunkserver/config_info.h b/src/chunkserver/config_info.h
@@ -14,6 +14,7 @@
 #include "src/fs/local_filesystem.h"
 #include "src/chunkserver/trash.h"
 #include "src/chunkserver/inflight_throttle.h"
+#include "include/chunkserver/chunkserver_common.h"
 
 namespace curve {
 namespace chunkserver {
@@ -83,6 +84,9 @@ struct CopysetNodeOptions {
     // 一段时间后实际回收物理空间
     std::shared_ptr<Trash> trash;
 
+    // snapshot流控
+    scoped_refptr<SnapshotThrottle> *snapshotThrottle;
+
     CopysetNodeOptions();
 };
 

diff --git a/test/chunkserver/chunkserver.conf.0 b/test/chunkserver/chunkserver.conf.0
@@ -23,6 +23,8 @@ mds.heartbeat_timeout=5000
 chunkserver.stor_uri=local://./0/
 chunkserver.meta_uri=local://./0/chunkserver.dat
 chunkserver.disk_type=nvme
+chunkserver.snapshot_throttle_throughput_bytes=41943040
+chunkserver.snapshot_throttle_check_cycles=4
 
 #
 # Testing purpose settings
@@ -91,4 +93,4 @@ chunkfilepool.retry_times=5
 # trash settings
 #
 trash.expire_afterSec=120
-trash.scan_periodSec=60
+trash.scan_periodSec=60
diff --git a/test/chunkserver/chunkserver.conf.1 b/test/chunkserver/chunkserver.conf.1
@@ -23,6 +23,8 @@ mds.heartbeat_timeout=5000
 chunkserver.stor_uri=local://./1/
 chunkserver.meta_uri=local://./1/chunkserver.dat
 chunkserver.disk_type=nvme
+chunkserver.snapshot_throttle_throughput_bytes=41943040
+chunkserver.snapshot_throttle_check_cycles=4
 
 #
 # Testing purpose settings

diff --git a/test/chunkserver/chunkserver.conf.2 b/test/chunkserver/chunkserver.conf.2
@@ -23,6 +23,8 @@ mds.heartbeat_timeout=5000
 chunkserver.stor_uri=local://./2/
 chunkserver.meta_uri=local://./2/chunkserver.dat
 chunkserver.disk_type=nvme
+chunkserver.snapshot_throttle_throughput_bytes=41943040
+chunkserver.snapshot_throttle_check_cycles=4
 
 #
 # Testing purpose settings