Skip to content

Commit 498463a

Browse files
周文煜Gerrit Code Review
周文煜
authored and
Gerrit Code Review
committed
Merge "Chunkserver Registration/Copyset recreation/Heartbeat" into feature
2 parents be170d2 + 7f839e7 commit 498463a

34 files changed

+3255
-303
lines changed

‎conf/chunkserver.conf.example

+26-8
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,33 @@
11
#
22
# Global settings
33
#
4-
global.logfile=./chunkserver.log
4+
#global.logfile=/root/data/log/0/chunkserver.log
55
global.ip=127.0.0.1
66
global.port=8200
7-
global.chunk_size=4096
7+
global.chunk_size=16777216
88
global.meta_page_size=4096
99

10+
#
11+
# MDS settings
12+
#
13+
mds.ip=127.0.0.1
14+
mds.port=6666
15+
mds.register_retries=100
16+
mds.register_timeout=1000
17+
mds.heartbeat_interval=1
18+
mds.heartbeat_timeout=5000
19+
20+
#
21+
# Chunkserver settings
22+
#
23+
chunkserver.stor_uri=local://./0/
24+
chunkserver.meta_uri=local://./0/chunkserver.dat
25+
chunkserver.disk_type=nvme
26+
1027
#
1128
# Testing purpose settings
1229
#
13-
test.create_testcopyset=true
30+
test.create_testcopyset=false
1431
test.testcopyset_poolid=666
1532
test.testcopyset_copysetid=888888
1633
test.testcopyset_conf=127.0.0.1:8200:0,127.0.0.1:8201:0,127.0.0.1:8202:0
@@ -24,10 +41,11 @@ copyset.log_applied_task=false
2441
copyset.election_timeout_ms=5000
2542
copyset.snapshot_interval=30
2643
copyset.catchup_margin=50
27-
copyset.chunk_data_uri=local://./3
28-
copyset.raft_log_uri=local://./3
29-
copyset.raft_meta_uri=local://./3
30-
copyset.raft_snapshot_uri=local://./3
44+
copyset.chunk_data_uri=local://./0/copysets
45+
copyset.raft_log_uri=local://./0/copysets
46+
copyset.raft_meta_uri=local://./0/copysets
47+
copyset.raft_snapshot_uri=local://./0/copysets
48+
copyset.recycler_uri=local://./0/recycler
3149

3250
#
3351
# Storage engine settings
@@ -48,7 +66,7 @@ concurrentapply.queuedepth=1
4866
# Chunkfile pool
4967
#
5068
chunkfilepool.enable_get_chunk_from_pool=false
51-
chunkfilepool.chunk_file_pool_dir=./
69+
chunkfilepool.chunk_file_pool_dir=./0/
5270
#chunkfilepool.meta_path=./chunkfilepool.meta
5371
chunkfilepool.cpmeta_file_size=4096
5472
chunkfilepool.retry_times=5

‎deploy/local/chunkserver/conf/chunkserver.conf.0

+24-6
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,33 @@
11
#
22
# Global settings
33
#
4-
global.logfile=./chunkserver.log
4+
#global.logfile=/root/data/log/0/chunkserver.log
55
global.ip=127.0.0.1
66
global.port=8200
77
global.chunk_size=16777216
88
global.meta_page_size=4096
99

10+
#
11+
# MDS settings
12+
#
13+
mds.ip=127.0.0.1
14+
mds.port=6666
15+
mds.register_retries=100
16+
mds.register_timeout=1000
17+
mds.heartbeat_interval=1
18+
mds.heartbeat_timeout=5000
19+
20+
#
21+
# Chunkserver settings
22+
#
23+
chunkserver.stor_uri=local://./0/
24+
chunkserver.meta_uri=local://./0/chunkserver.dat
25+
chunkserver.disk_type=nvme
26+
1027
#
1128
# Testing purpose settings
1229
#
13-
test.create_testcopyset=true
30+
test.create_testcopyset=false
1431
test.testcopyset_poolid=666
1532
test.testcopyset_copysetid=888888
1633
test.testcopyset_conf=127.0.0.1:8200:0,127.0.0.1:8201:0,127.0.0.1:8202:0
@@ -24,10 +41,11 @@ copyset.log_applied_task=false
2441
copyset.election_timeout_ms=5000
2542
copyset.snapshot_interval=30
2643
copyset.catchup_margin=50
27-
copyset.chunk_data_uri=local://./0
28-
copyset.raft_log_uri=local://./0
29-
copyset.raft_meta_uri=local://./0
30-
copyset.raft_snapshot_uri=local://./0
44+
copyset.chunk_data_uri=local://./0/copysets
45+
copyset.raft_log_uri=local://./0/copysets
46+
copyset.raft_meta_uri=local://./0/copysets
47+
copyset.raft_snapshot_uri=local://./0/copysets
48+
copyset.recycler_uri=local://./0/recycler
3149

3250
#
3351
# Storage engine settings

‎deploy/local/chunkserver/conf/chunkserver.conf.1

+24-6
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,33 @@
11
#
22
# Global settings
33
#
4-
global.logfile=./chunkserver.log
4+
#global.logfile=/root/data/log/1/chunkserver.log
55
global.ip=127.0.0.1
66
global.port=8201
77
global.chunk_size=16777216
88
global.meta_page_size=4096
99

10+
#
11+
# MDS settings
12+
#
13+
mds.ip=127.0.0.1
14+
mds.port=6666
15+
mds.register_retries=100
16+
mds.register_timeout=1000
17+
mds.heartbeat_interval=1
18+
mds.heartbeat_timeout=5000
19+
20+
#
21+
# Chunkserver settings
22+
#
23+
chunkserver.stor_uri=local://./1/
24+
chunkserver.meta_uri=local://./1/chunkserver.dat
25+
chunkserver.disk_type=nvme
26+
1027
#
1128
# Testing purpose settings
1229
#
13-
test.create_testcopyset=true
30+
test.create_testcopyset=false
1431
test.testcopyset_poolid=666
1532
test.testcopyset_copysetid=888888
1633
test.testcopyset_conf=127.0.0.1:8200:0,127.0.0.1:8201:0,127.0.0.1:8202:0
@@ -24,10 +41,11 @@ copyset.log_applied_task=false
2441
copyset.election_timeout_ms=5000
2542
copyset.snapshot_interval=30
2643
copyset.catchup_margin=50
27-
copyset.chunk_data_uri=local://./1
28-
copyset.raft_log_uri=local://./1
29-
copyset.raft_meta_uri=local://./1
30-
copyset.raft_snapshot_uri=local://./1
44+
copyset.chunk_data_uri=local://./1/copysets
45+
copyset.raft_log_uri=local://./1/copysets
46+
copyset.raft_meta_uri=local://./1/copysets
47+
copyset.raft_snapshot_uri=local://./1/copysets
48+
copyset.recycler_uri=local://./1/recycler
3149

3250
#
3351
# Storage engine settings

‎deploy/local/chunkserver/conf/chunkserver.conf.2

+25-6
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,33 @@
11
#
22
# Global settings
33
#
4-
global.logfile=./chunkserver.log
4+
#global.logfile=/root/data/log/1/chunkserver.log
55
global.ip=127.0.0.1
66
global.port=8202
77
global.chunk_size=16777216
88
global.meta_page_size=4096
99

10+
#
11+
# MDS settings
12+
#
13+
mds.ip=127.0.0.1
14+
mds.port=6666
15+
mds.register_retries=100
16+
mds.register_timeout=1000
17+
mds.heartbeat_interval=1
18+
mds.heartbeat_timeout=5000
19+
20+
#
21+
# Chunkserver settings
22+
#
23+
chunkserver.stor_uri=local://./2/
24+
chunkserver.meta_uri=local://./2/chunkserver.dat
25+
chunkserver.disk_type=nvme
26+
1027
#
1128
# Testing purpose settings
1229
#
13-
test.create_testcopyset=true
30+
test.create_testcopyset=false
1431
test.testcopyset_poolid=666
1532
test.testcopyset_copysetid=888888
1633
test.testcopyset_conf=127.0.0.1:8200:0,127.0.0.1:8201:0,127.0.0.1:8202:0
@@ -24,10 +41,11 @@ copyset.log_applied_task=false
2441
copyset.election_timeout_ms=5000
2542
copyset.snapshot_interval=30
2643
copyset.catchup_margin=50
27-
copyset.chunk_data_uri=local://./2
28-
copyset.raft_log_uri=local://./2
29-
copyset.raft_meta_uri=local://./2
30-
copyset.raft_snapshot_uri=local://./2
44+
copyset.chunk_data_uri=local://./2/copysets
45+
copyset.raft_log_uri=local://./2/copysets
46+
copyset.raft_meta_uri=local://./2/copysets
47+
copyset.raft_snapshot_uri=local://./2/copysets
48+
copyset.recycler_uri=local://./2/recycler
3149

3250
#
3351
# Storage engine settings
@@ -52,3 +70,4 @@ chunkfilepool.chunk_file_pool_dir=./2/
5270
#chunkfilepool.meta_path=./2/chunkfilepool.meta
5371
chunkfilepool.cpmeta_file_size=4096
5472
chunkfilepool.retry_times=5
73+

‎deploy/local/chunkserver/start_chunkservers_locally.sh

+9-4
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
#!/bin/sh
22

33
home=deploy/local/chunkserver
4+
loghome=log
45
#home=.
56
conf=${home}/conf
6-
log=${home}/log
77
bin=bazel-bin/src/chunkserver
88
#bin=.
99

10-
${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=0 -conf=${conf}/chunkserver.conf.0 > ${log}/chunkserver.log.0 2>&1 &
11-
${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=0 -conf=${conf}/chunkserver.conf.1 > ${log}/chunkserver.log.1 2>&1 &
12-
${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=0 -conf=${conf}/chunkserver.conf.2 > ${log}/chunkserver.log.2 2>&1 &
10+
[ -f ${loghome} ] || mkdir -p ${loghome}
11+
[ -f ${loghome}/0 ] || mkdir -p ${loghome}/0
12+
[ -f ${loghome}/1 ] || mkdir -p ${loghome}/1
13+
[ -f ${loghome}/2 ] || mkdir -p ${loghome}/2
14+
15+
${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=0 -v 19 -conf=${conf}/chunkserver.conf.0 2>${loghome}/0/chunkserver.err &
16+
${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=0 -v 19 -conf=${conf}/chunkserver.conf.1 2>${loghome}/1/chunkserver.err &
17+
${bin}/chunkserver -bthread_concurrency=18 -raft_max_segment_size=8388608 -raft_sync=true -minloglevel=0 -v 19 -conf=${conf}/chunkserver.conf.2 2>${loghome}/2/chunkserver.err &

‎include/chunkserver/chunkserver_common.h

+50-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <braft/file_system_adaptor.h>
1313

1414
#include <cstdint>
15+
#include <string>
1516

1617
namespace curve {
1718
namespace chunkserver {
@@ -24,7 +25,10 @@ using SnapshotID = uint64_t;
2425
using SequenceNum = uint64_t;
2526

2627
using ChunkSizeType = uint32_t;
27-
using PageSizeType = uint32_t;
28+
using PageSizeType = uint32_t;
29+
30+
using GroupNid = uint64_t;
31+
using ChunkServerID = uint32_t;
2832

2933
// braft
3034
using Configuration = braft::Configuration;
@@ -36,6 +40,51 @@ using FileSystemAdaptor = braft::FileSystemAdaptor;
3640
using DirReader = braft::DirReader;
3741
using PosixFileSystemAdaptor = braft::PosixFileSystemAdaptor;
3842

43+
/**
44+
* 将(LogicPoolID, CopysetID)二元组转换成数字格式的复制组ID,格式如下：
45+
* | group id |
46+
* | 32 | 32 |
47+
* | logic pool id | copyset id |
48+
*/
49+
inline GroupNid ToGroupNid(const LogicPoolID &logicPoolId,
50+
const CopysetID &copysetId) {
51+
return (static_cast<uint64_t>(logicPoolId) << 32) | copysetId;
52+
}
53+
/**
54+
* 将(LogicPoolID, CopysetID)二元组转换成字符串格式的复制组ID
55+
*/
56+
inline GroupId ToGroupId(const LogicPoolID &logicPoolId,
57+
const CopysetID &copysetId) {
58+
return std::to_string(ToGroupNid(logicPoolId, copysetId));
59+
}
60+
#define ToBraftGroupId ToGroupId
61+
62+
/**
63+
* 从数字格式的复制组ID中解析LogicPoolID
64+
*/
65+
inline LogicPoolID GetPoolID(const GroupNid &groupId) {
66+
return groupId >> 32;
67+
}
68+
/**
69+
* 从数字格式的复制组ID中解析CopysetID
70+
*/
71+
inline CopysetID GetCopysetID(const GroupNid &groupId) {
72+
return groupId & (((uint64_t)1 << 32) - 1);
73+
}
74+
75+
/* 格式输出 group id 的 字符串 (logicPoolId, copysetId) */
76+
inline std::string ToGroupIdString(const LogicPoolID &logicPoolId,
77+
const CopysetID &copysetId) {
78+
std::string groupIdString;
79+
groupIdString.append("(");
80+
groupIdString.append(std::to_string(logicPoolId));
81+
groupIdString.append(", ");
82+
groupIdString.append(std::to_string(copysetId));
83+
groupIdString.append(")");
84+
return groupIdString;
85+
}
86+
#define ToGroupIdStr ToGroupIdString
87+
3988
} // namespace chunkserver
4089
} // namespace curve
4190

‎proto/chunkserver.proto

+8-26
Original file line numberDiff line numberDiff line change
@@ -2,43 +2,25 @@ syntax="proto2";
22
package curve.chunkserver;
33
option cc_generic_services = true;
44

5-
// ChunkServer service
6-
enum CHUNKSERVER_OP_TYPE {
7-
CHUNKSERVER_OP_TYPE_ENABLE = 0;
8-
CHUNKSERVER_OP_TYPE_DISABLE = 1;
9-
CHUNKSERVER_OP_TYPE_GETSTATE = 2;
10-
CHUNKSERVER_OP_TYPE_GETINFO = 3;
5+
message ChunkServerMetadata {
6+
required uint32 version = 1;
7+
required uint32 id = 2;
8+
required string token = 3;
9+
required uint32 checksum = 4;
1110
};
1211

13-
message ChunkServerRequest {
14-
required CHUNKSERVER_OP_TYPE op = 1;
12+
message ChunkServerStopRequest {
1513
};
1614

1715
enum CHUNKSERVER_OP_STATUS {
1816
CHUNKSERVER_OP_STATUS_SUCCESS = 0;
1917
CHUNKSERVER_OP_STATUS_FAILURE_UNKNOWN = 1;
2018
};
2119

22-
enum CHUNKSERVER_STATE {
23-
CHUNKSERVER_STATE_OK = 0;
24-
CHUNKSERVER_STATE_DISABLED = 1;
25-
};
26-
27-
message ChunkServerInfo {
28-
required int32 id = 1;
29-
required string name = 2;
30-
required CHUNKSERVER_STATE state = 3;
31-
required int32 capacity = 4;
32-
required int32 available = 5;
33-
};
34-
35-
message ChunkServerResponse {
20+
message ChunkServerStopResponse {
3621
required CHUNKSERVER_OP_STATUS status = 1;
37-
optional CHUNKSERVER_STATE state = 2;
38-
optional ChunkServerInfo info = 3;
3922
};
4023

4124
service ChunkServerService {
42-
rpc GetState(ChunkServerRequest) returns(ChunkServerResponse);
43-
rpc GetInfo(ChunkServerRequest) returns(ChunkServerResponse);
25+
rpc Stop(ChunkServerStopRequest) returns(ChunkServerStopResponse);
4426
};

‎src/chunkserver/BUILD

+4-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ COPTS = [
33
"-DOS_LINUX",
44
"-DSNAPPY",
55
"-DHAVE_SSE42",
6-
"-DNDEBUG",
76
"-fno-omit-frame-pointer",
87
"-momit-leaf-frame-pointer",
98
"-msse4.2",
@@ -48,6 +47,8 @@ cc_library(
4847
"//external:leveldb",
4948
"//external:protobuf",
5049
"//proto:chunkserver-cc-protos",
50+
"//proto:topology_cc_proto",
51+
"//proto:heartbeat_cc_proto",
5152
"//src/chunkserver/chunkserverStorage:chunkserver-sfsadaptor-lib",
5253
"//src/chunkserver/datastore:chunkserver_datastore",
5354
"//src/common:curve_common",
@@ -87,6 +88,8 @@ cc_library(
8788
"//external:leveldb",
8889
"//external:protobuf",
8990
"//proto:chunkserver-cc-protos",
91+
"//proto:heartbeat_cc_proto",
92+
"//proto:topology_cc_proto",
9093
"//src/chunkserver/chunkserverStorage:chunkserver-sfsadaptor-lib",
9194
"//src/chunkserver/datastore:chunkserver_datastore",
9295
"//src/common:curve_common",

0 commit comments

Comments
 (0)