forked from opencurve/curve
-
Notifications
You must be signed in to change notification settings - Fork 0
/
curve_fs_robot.txt
311 lines (273 loc) · 8.13 KB
/
curve_fs_robot.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
*** Settings ***
Library Collections
#Library RequestsLibrary
Library Process
#Library SSHLibrary
Library Resources/keywords/fs_fault_inject.py
Library Resources/keywords/fault_inject.py
Library Resources/keywords/deploy.py
Suite Setup init failover cluster
Suite Teardown clean failover env
*** Variables ***
${fio_vdsize} 10
${vdbench_vdsize} 10
${vm_iops_limit} 8000
${chunk_num} 0
*** Test Cases ***
begin pjdtest
[Tags] P2 base interface pjdtest
${pjdfs} evaluate str("failover-pjdtest")
mount test dir ${pjdfs}
start pjdtest ${pjdfs}
inject kill one mds
[Tags] P0 base first release failover
${num} evaluate int(1)
test kill process mds ${num}
check fuse iops
sleep 3
test start process mds
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] test start process mds
inject kill one metaserver
[Tags] P0 base first release failover
${num} evaluate int(1)
test kill process metaserver ${num}
check fuse iops
sleep 3
test start process metaserver
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] test start process metaserver
inject kill one etcd
[Tags] P0 base first release failover
${num} evaluate int(1)
test kill process etcd ${num}
check fuse iops
sleep 3
test start process etcd
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] test start process etcd
test fs mds loss package 5%
[Tags] P1 base first release failover
${percent} evaluate int(5)
test fs process loss package mds ${percent}
check fuse mount success
check fuse iops
[Teardown] wait fs cluster ok
test fs metaserver loss package 5%
[Tags] P1 base first release failover
${percent} evaluate int(5)
test fs process loss package metaserver ${percent}
check fuse mount success
check fuse iops
[Teardown] check fs cluster ok
test fs client loss package 5%
[Tags] P1 base first release failover
${percent} evaluate int(5)
test fs process loss package fuseclient ${percent}
check fuse mount success
check fuse iops
[Teardown] check fs cluster ok
test fs etcd loss package 5%
[Tags] P1 base first release failover
${percent} evaluate int(5)
test fs process loss package etcd ${percent}
check fuse mount success
check fuse iops
[Teardown] check fs cluster ok
test fs mds delay package 100ms
[Tags] P1 base first release failover
${ms} evaluate int(100)
test fs process delay package mds ${ms}
check fuse mount success
check fuse iops
[Teardown] check fs cluster ok
test fs metaserver delay package 100ms
[Tags] P1 base first release failover
${ms} evaluate int(100)
test fs process delay package metaserver ${ms}
check fuse mount success
check fuse iops
[Teardown] check fs cluster ok
test fs etcd delay package 100ms
[Tags] P1 base first release failover
${ms} evaluate int(100)
test fs process delay package etcd ${ms}
check fuse mount success
check fuse iops
[Teardown] check fs cluster ok
test fs client delay package 100ms
[Tags] P1 base first release failover
${ms} evaluate int(100)
test fs process delay package fuseclient ${ms}
check fuse mount success
check fuse iops
[Teardown] check fs cluster ok
inject fuse client cpu stress 95%
[Tags] P1 base first release failover
${cpu_stress} evaluate int(95)
${ssh} test fuse client cpu stress ${cpu_stress}
sleep 5
del cpu stress ${ssh}
check fuse mount success
check fuse iops
[Teardown] del cpu stress ${ssh}
inject fuse client mem stress 90%
[Tags] P1 base first release failover
${mem_stress} evaluate int(75)
${ssh} test fuse client mem stress ${mem_stress}
sleep 5
del mem stress ${ssh}
check fuse mount success
check fuse iops
[Teardown] wait fs cluster ok
test metaserver copyset migrates out in
[Tags] P1 base first release failover
test out metaserver copyset
test in metaserver copyset
check fuse mount success
test ipmitool restart metaserver node
[Tags] P1 base first release failover ipmitool
test ipmitool restart metaserver
sleep 60
check fuse mount success
check fuse iops
[Teardown] wait fs cluster ok
test check pjdtest result
[Tags] P2 base interface pjdtest
check pjdtest result
test loop mount umount
[Tags] P0 base first release failover
sleep 30
stop loop mount
check fuse mount success
inject kill two mds
[Tags] P1 base first release failover
${num} evaluate int(2)
test kill process mds ${num}
sleep 5
test start process mds
sleep 5
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] test start process mds
inject kill all mds
[Tags] P1 base first release failover
${num} evaluate int(3)
test kill process mds ${num}
sleep 5
test start process mds
sleep 5
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] wait fs cluster ok
inject kill two metaserver
[Tags] P1 base first release failover
${num} evaluate int(2)
test kill process metaserver ${num}
sleep 5
test start process metaserver
sleep 5
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] wait fs cluster ok
inject kill all metaserver
[Tags] P1 base first release failover
${num} evaluate int(3)
test kill process metaserver ${num}
sleep 5
test start process metaserver
sleep 5
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] wait fs cluster ok
inject kill two etcd
[Tags] P1 base first release failover
${num} evaluate int(2)
test kill process etcd ${num}
sleep 5
test start process etcd
sleep 5
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] test start process etcd
inject kill all etcd
[Tags] P1 base first release failover
${num} evaluate int(3)
test kill process etcd ${num}
sleep 5
test start process etcd
sleep 5
check fuse mount success
check fuse iops
check fs cluster ok
[Teardown] test start process etcd
check cp data consistency
[Tags] P0 base first release failover
checksum data
check fuse mount success
wait op finish
check mv data consistency
[Tags] P1 base first release failover
mv data
checksum mv data
check fuse mount success
check cto file md5
[Tags] P0 base first release failover
stop cto file md5check
check umount date consistency
[Tags] P1 base first release failover
sleep 60
get test dir file md5
umount test dir test2
sleep 120
wait fuse exit failover/test2
mount test dir test2
sleep 60
check test dir file md5
check cto dir file md5 test5
# [Teardown] wait fs cluster ok
*** Keywords ***
init failover cluster
clean env
destroy curvefs
clean log
deploy all servers
sleep 30
check fs cluster ok
mount test dir
sleep 40
check fuse mount success
write check data
cp check data
start fs op test
mount fuse cto test
clean log
clean fs kernel log
clean corefile
start fs op test
start fs vdbench
start fs fio
loop mount umount
sleep 60
mount fuse cto test
mount test dir test1 test4
mount test dir test2 test5
start cto file md5check
clean failover env
check log
# destroy curvefs
check log
check fs io error
check vdbench output
check corefile