Skip to content

Commit 2009d2e

Browse files
Vigor-jpgxu-chaojie
Vigor-jpg
authored and
xu-chaojie
committed
Chunk pool format asyn
Signed-off-by: yyyyufeng <[email protected]>
1 parent ba010be commit 2009d2e

35 files changed

+1036
-242
lines changed

‎conf/chunkserver.conf

+17-1
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ rconcurrentapply.queuedepth=1
194194
# 是否开启从chunkfilepool获取chunk，一般是true
195195
chunkfilepool.enable_get_chunk_from_pool=true
196196
# chunkfilepool目录
197-
chunkfilepool.chunk_file_pool_dir=./0/ # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__
197+
chunkfilepool.chunk_file_pool_dir=./0/chunks # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__
198198
# chunkfilepool meta文件路径
199199
chunkfilepool.meta_path=./chunkfilepool.meta # __CURVEADM_TEMPLATE__ ${prefix}/data/chunkfilepool.meta __CURVEADM_TEMPLATE__
200200
# chunkfilepool meta文件大小
@@ -207,6 +207,14 @@ chunkfilepool.clean.enable=true
207207
chunkfilepool.clean.bytes_per_write=4096
208208
# The throttle iops for cleaning chunk (4KB/IO)
209209
chunkfilepool.clean.throttle_iops=500
210+
# Whether allocate filePool by percent of disk size.
211+
chunkfilepool.allocated_by_percent=true
212+
# Preallocate storage percent of total disk
213+
chunkfilepool.allocate_percent=80
214+
# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB)
215+
chunkfilepool.chunk_file_pool_size=1GB
216+
# The thread num for format chunks
217+
chunkfilepool.thread_num=1
210218

211219
#
212220
# WAL file pool
@@ -229,6 +237,14 @@ walfilepool.metapage_size=4096
229237
walfilepool.meta_file_size=4096
230238
# WAL filepool get chunk最大重试次数
231239
walfilepool.retry_times=5
240+
# Whether allocate filePool by percent of disk size.
241+
walfilepool.allocated_by_percent=true
242+
# Preallocate storage percent of total disk
243+
walfilepool.allocate_percent=90
244+
# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB)
245+
walfilepool.wal_file_pool_size=0
246+
# The thread num for format chunks
247+
walfilepool.thread_num=1
232248

233249
#
234250
# trash settings

‎conf/chunkserver.conf.example

+17-1
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ rconcurrentapply.queuedepth=1
186186
# 是否开启从chunkfilepool获取chunk，一般是true
187187
chunkfilepool.enable_get_chunk_from_pool=true
188188
# chunkfilepool目录
189-
chunkfilepool.chunk_file_pool_dir=./0/
189+
chunkfilepool.chunk_file_pool_dir=./0/chunks
190190
# chunkfilepool meta文件路径
191191
#chunkfilepool.meta_path=./chunkfilepool.meta
192192
# chunkfilepool meta文件大小
@@ -199,6 +199,14 @@ chunkfilepool.clean.enable=true
199199
chunkfilepool.clean.bytes_per_write=4096
200200
# The throttle iops for cleaning chunk (4KB/IO)
201201
chunkfilepool.clean.throttle_iops=500
202+
# Whether allocate filePool by percent of disk size.
203+
chunkfilepool.allocated_by_percent=true
204+
# Preallocate storage percent of total disk
205+
chunkfilepool.allocate_percent=80
206+
# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB)
207+
chunkfilepool.chunk_file_pool_size=1GB
208+
# The thread num for format chunks
209+
chunkfilepool.thread_num=1
202210

203211
#
204212
# WAL file pool
@@ -221,6 +229,14 @@ walfilepool.metapage_size=4096
221229
walfilepool.meta_file_size=4096
222230
# WAL filepool get chunk最大重试次数
223231
walfilepool.retry_times=5
232+
# Whether allocate filePool by percent of disk size.
233+
walfilepool.allocated_by_percent=true
234+
# Preallocate storage percent of total disk
235+
walfilepool.allocate_percent=10
236+
# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB)
237+
walfilepool.wal_file_pool_size=0
238+
# The thread num for format chunks
239+
walfilepool.thread_num=1
224240

225241
#
226242
# trash settings

‎proto/heartbeat.proto

+2
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ message ChunkServerStatisticInfo {
8989
required uint64 chunkSizeTrashedBytes = 7;
9090
// chunkfilepool的大小
9191
optional uint64 chunkFilepoolSize = 8;
92+
// percentage of chunkfilepool formatting
93+
optional uint32 chunkFilepoolFormatPercent = 9;
9294
};
9395

9496
message ChunkServerHeartbeatRequest {

‎proto/topology.proto

+14
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,19 @@ message GetCopySetsInChunkServerRequest {
501501
optional uint32 port = 3;
502502
}
503503

504+
message ChunkFormatStatus {
505+
required string ip = 1;
506+
required uint32 port = 2;
507+
required uint32 chunkServerID = 3;
508+
required uint32 formatPercent = 4;
509+
}
510+
511+
message ListChunkFormatStatusRequest {}
512+
513+
message ListChunkFormatStatusResponse {
514+
repeated ChunkFormatStatus chunkFormatStatus = 1;
515+
}
516+
504517
message GetCopySetsInChunkServerResponse {
505518
required sint32 statusCode = 1;
506519
repeated common.CopysetInfo copysetInfos = 2;
@@ -595,4 +608,5 @@ service TopologyService {
595608
rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse);
596609
rpc SetCopysetsAvailFlag(SetCopysetsAvailFlagRequest) returns (SetCopysetsAvailFlagResponse);
597610
rpc ListUnAvailCopySets(ListUnAvailCopySetsRequest) returns (ListUnAvailCopySetsResponse);
611+
rpc ListChunkFormatStatus(ListChunkFormatStatusRequest) returns (ListChunkFormatStatusResponse);
598612
}

‎src/chunkserver/chunkserver.cpp

+101-15
Original file line numberDiff line numberDiff line change
@@ -20,30 +20,31 @@
2020
* Author: lixiaocui
2121
*/
2222

23-
#include <glog/logging.h>
23+
#include "src/chunkserver/chunkserver.h"
2424

25-
#include <butil/endpoint.h>
2625
#include <braft/builtin_service_impl.h>
2726
#include <braft/raft_service.h>
2827
#include <braft/storage.h>
28+
#include <butil/endpoint.h>
29+
#include <glog/logging.h>
2930

3031
#include <memory>
3132

32-
#include "src/chunkserver/chunkserver.h"
33-
#include "src/chunkserver/chunkserver_metrics.h"
34-
#include "src/chunkserver/chunkserver_service.h"
35-
#include "src/chunkserver/copyset_service.h"
36-
#include "src/chunkserver/chunk_service.h"
3733
#include "src/chunkserver/braft_cli_service.h"
3834
#include "src/chunkserver/braft_cli_service2.h"
35+
#include "src/chunkserver/chunk_service.h"
3936
#include "src/chunkserver/chunkserver_helper.h"
40-
#include "src/common/concurrent/task_thread_pool.h"
41-
#include "src/common/uri_parser.h"
42-
#include "src/chunkserver/raftsnapshot/curve_snapshot_attachment.h"
37+
#include "src/chunkserver/chunkserver_metrics.h"
38+
#include "src/chunkserver/chunkserver_service.h"
39+
#include "src/chunkserver/copyset_service.h"
40+
#include "src/chunkserver/raftlog/curve_segment_log_storage.h"
4341
#include "src/chunkserver/raftsnapshot/curve_file_service.h"
42+
#include "src/chunkserver/raftsnapshot/curve_snapshot_attachment.h"
4443
#include "src/chunkserver/raftsnapshot/curve_snapshot_storage.h"
45-
#include "src/chunkserver/raftlog/curve_segment_log_storage.h"
44+
#include "src/common/bytes_convert.h"
45+
#include "src/common/concurrent/task_thread_pool.h"
4646
#include "src/common/curve_version.h"
47+
#include "src/common/uri_parser.h"
4748

4849
using ::curve::fs::LocalFileSystem;
4950
using ::curve::fs::LocalFileSystemOption;
@@ -65,6 +66,10 @@ DEFINE_string(raftSnapshotUri, "curve://./0/copysets", "raft snapshot uri");
6566
DEFINE_string(raftLogUri, "curve://./0/copysets", "raft log uri");
6667
DEFINE_string(recycleUri, "local://./0/recycler" , "recycle uri");
6768
DEFINE_string(chunkFilePoolDir, "./0/", "chunk file pool location");
69+
DEFINE_int32(chunkFilePoolAllocatedPercent, 80,
70+
"format percent for chunkfillpool.");
71+
DEFINE_uint32(chunkFormatThreadNum, 1,
72+
"number of threads while file pool formatting");
6873
DEFINE_string(chunkFilePoolMetaPath,
6974
"./chunkfilepool.meta", "chunk file pool meta path");
7075
DEFINE_string(logPath, "./0/chunkserver.log-", "log file path");
@@ -479,8 +484,6 @@ void ChunkServer::Stop() {
479484
brpc::AskToQuit();
480485
}
481486

482-
483-
484487
void ChunkServer::InitChunkFilePoolOptions(
485488
common::Configuration *conf, FilePoolOptions *chunkFilePoolOptions) {
486489
LOG_IF(FATAL, !conf->GetUInt32Value("global.chunk_size",
@@ -513,13 +516,56 @@ void ChunkServer::InitChunkFilePoolOptions(
513516
"chunkfilepool.meta_path", &metaUri));
514517
::memcpy(
515518
chunkFilePoolOptions->metaPath, metaUri.c_str(), metaUri.size());
519+
520+
std::string chunkFilePoolUri;
521+
LOG_IF(FATAL, !conf->GetStringValue("chunkfilepool.chunk_file_pool_dir",
522+
&chunkFilePoolUri));
523+
524+
::memcpy(chunkFilePoolOptions->filePoolDir, chunkFilePoolUri.c_str(),
525+
chunkFilePoolUri.size());
526+
std::string pool_size;
527+
LOG_IF(FATAL, !conf->GetStringValue(
528+
"chunkfilepool.chunk_file_pool_size", &pool_size));
529+
LOG_IF(FATAL, !curve::common::ToNumbericByte(
530+
pool_size, &chunkFilePoolOptions->filePoolSize));
531+
LOG_IF(FATAL,
532+
!conf->GetBoolValue("chunkfilepool.allocated_by_percent",
533+
&chunkFilePoolOptions->allocatedByPercent));
534+
LOG_IF(FATAL,
535+
!conf->GetUInt32Value("chunkfilepool.allocate_percent",
536+
&chunkFilePoolOptions->allocatedPercent));
537+
LOG_IF(FATAL, !conf->GetUInt32Value(
538+
"chunkfilepool.chunk_file_pool_format_thread_num",
539+
&chunkFilePoolOptions->formatThreadNum));
516540
LOG_IF(FATAL, !conf->GetBoolValue("chunkfilepool.clean.enable",
517541
&chunkFilePoolOptions->needClean));
518-
LOG_IF(FATAL, !conf->GetUInt32Value("chunkfilepool.clean.bytes_per_write", // NOLINT
519-
&chunkFilePoolOptions->bytesPerWrite));
542+
LOG_IF(FATAL,
543+
!conf->GetUInt32Value("chunkfilepool.clean.bytes_per_write",
544+
&chunkFilePoolOptions->bytesPerWrite));
520545
LOG_IF(FATAL, !conf->GetUInt32Value("chunkfilepool.clean.throttle_iops",
521546
&chunkFilePoolOptions->iops4clean));
522547

548+
std::string copysetUri;
549+
LOG_IF(FATAL,
550+
!conf->GetStringValue("copyset.raft_snapshot_uri", &copysetUri));
551+
curve::common::UriParser::ParseUri(copysetUri,
552+
&chunkFilePoolOptions->copysetDir);
553+
554+
std::string recycleUri;
555+
LOG_IF(FATAL,
556+
!conf->GetStringValue("copyset.recycler_uri", &recycleUri));
557+
curve::common::UriParser::ParseUri(recycleUri,
558+
&chunkFilePoolOptions->recycleDir);
559+
560+
bool useChunkFilePoolAsWalPool;
561+
LOG_IF(FATAL, !conf->GetBoolValue("walfilepool.use_chunk_file_pool",
562+
&useChunkFilePoolAsWalPool));
563+
564+
chunkFilePoolOptions->isAllocated = [=](const std::string& filename) {
565+
return Trash::IsChunkOrSnapShotFile(filename) ||
566+
(useChunkFilePoolAsWalPool && Trash::IsWALFile(filename));
567+
};
568+
523569
if (0 == chunkFilePoolOptions->bytesPerWrite
524570
|| chunkFilePoolOptions->bytesPerWrite > 1 * 1024 * 1024
525571
|| 0 != chunkFilePoolOptions->bytesPerWrite % 4096) {
@@ -565,6 +611,36 @@ void ChunkServer::InitWalFilePoolOptions(
565611
std::string metaUri;
566612
LOG_IF(FATAL, !conf->GetStringValue(
567613
"walfilepool.meta_path", &metaUri));
614+
615+
std::string pool_size;
616+
LOG_IF(FATAL, !conf->GetStringValue("walfilepool.chunk_file_pool_size",
617+
&pool_size));
618+
LOG_IF(FATAL, !curve::common::ToNumbericByte(
619+
pool_size, &walPoolOptions->filePoolSize));
620+
LOG_IF(FATAL, !conf->GetUInt64Value("walfilepool.wal_file_pool_size",
621+
&walPoolOptions->filePoolSize));
622+
LOG_IF(FATAL, !conf->GetBoolValue("walfilepool.allocated_by_percent",
623+
&walPoolOptions->allocatedByPercent));
624+
LOG_IF(FATAL, !conf->GetUInt32Value("walfilepool.allocated_percent",
625+
&walPoolOptions->allocatedPercent));
626+
LOG_IF(FATAL, !conf->GetUInt32Value("walfilepool.thread_num",
627+
&walPoolOptions->formatThreadNum));
628+
629+
std::string copysetUri;
630+
LOG_IF(FATAL,
631+
!conf->GetStringValue("copyset.raft_log_uri", &copysetUri));
632+
curve::common::UriParser::ParseUri(copysetUri,
633+
&walPoolOptions->copysetDir);
634+
635+
std::string recycleUri;
636+
LOG_IF(FATAL,
637+
!conf->GetStringValue("copyset.recycler_uri", &recycleUri));
638+
curve::common::UriParser::ParseUri(recycleUri,
639+
&walPoolOptions->recycleDir);
640+
641+
walPoolOptions->isAllocated = [](const string& filename) {
642+
return Trash::IsWALFile(filename);
643+
};
568644
::memcpy(
569645
walPoolOptions->metaPath, metaUri.c_str(), metaUri.size());
570646
}
@@ -833,6 +909,16 @@ void ChunkServer::LoadConfigFromCmdline(common::Configuration *conf) {
833909
<< "chunkFilePoolDir must be set when run chunkserver in command.";
834910
}
835911

912+
if (GetCommandLineFlagInfo("chunkFilePoolAllocatedPercent", &info)) {
913+
conf->SetUInt32Value("chunkfilepool.allocate_percent",
914+
FLAGS_chunkFilePoolAllocatedPercent);
915+
}
916+
917+
if (GetCommandLineFlagInfo("chunkFormatThreadNum", &info)) {
918+
conf->SetUInt64Value("chunkfilepool.chunk_file_pool_format_thread_num",
919+
FLAGS_chunkFormatThreadNum);
920+
}
921+
836922
if (GetCommandLineFlagInfo("chunkFilePoolMetaPath", &info) &&
837923
!info.is_default) {
838924
conf->SetStringValue(

0 commit comments

Comments
 (0)