qa: more tries for mon tell when injecting msgr failures

With failure injection the default 2 tries isn't quite enough Signed-off-by: Sage Weil <[email protected]>
AsylumCorp · Oct 11, 2019 · 71d74aa · 71d74aa
1 parent cbaa6c0
commit 71d74aa
Show file tree

Hide file tree

Showing 38 changed files with 40 additions and 1 deletion.
diff --git a/qa/suites/fs/thrash/msgr-failures/osd-mds-delay.yaml b/qa/suites/fs/thrash/msgr-failures/osd-mds-delay.yaml
@@ -6,5 +6,6 @@ overrides:
         ms inject delay type: osd mds
         ms inject delay probability: .005
         ms inject delay max: 1
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/fsx/striping/default/msgr-failures/few.yaml b/qa/suites/krbd/fsx/striping/default/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/fsx/striping/default/msgr-failures/many.yaml b/qa/suites/krbd/fsx/striping/default/msgr-failures/many.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 500
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/fsx/striping/fancy/msgr-failures/few.yaml b/qa/suites/krbd/fsx/striping/fancy/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/rbd-nomount/msgr-failures/few.yaml b/qa/suites/krbd/rbd-nomount/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/rbd-nomount/msgr-failures/many.yaml b/qa/suites/krbd/rbd-nomount/msgr-failures/many.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 500
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/rbd/msgr-failures/few.yaml b/qa/suites/krbd/rbd/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/rbd/msgr-failures/many.yaml b/qa/suites/krbd/rbd/msgr-failures/many.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 500
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/singleton/msgr-failures/few.yaml b/qa/suites/krbd/singleton/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/singleton/msgr-failures/many.yaml b/qa/suites/krbd/singleton/msgr-failures/many.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 500
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/krbd/wac/wac/verify/many-resets.yaml b/qa/suites/krbd/wac/wac/verify/many-resets.yaml
@@ -3,6 +3,7 @@ overrides:
     conf:
       global:
         ms inject socket failures: 500
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
 tasks:

diff --git a/qa/suites/rados/basic/msgr-failures/few.yaml b/qa/suites/rados/basic/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/basic/msgr-failures/many.yaml b/qa/suites/rados/basic/msgr-failures/many.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 1500
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/monthrash/msgr-failures/few.yaml b/qa/suites/rados/monthrash/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/monthrash/msgr-failures/mon-delay.yaml b/qa/suites/rados/monthrash/msgr-failures/mon-delay.yaml
@@ -7,6 +7,7 @@ overrides:
         ms inject delay probability: .005
         ms inject delay max: 1
         ms inject internal delays: .002
+        mon client directed command retry: 5
       mgr:
         debug monc: 10
     log-whitelist:

diff --git a/qa/suites/rados/multimon/msgr-failures/few.yaml b/qa/suites/rados/multimon/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/multimon/msgr-failures/many.yaml b/qa/suites/rados/multimon/msgr-failures/many.yaml
@@ -3,6 +3,7 @@ overrides:
     conf:
       global:
         ms inject socket failures: 1000
+        mon client directed command retry: 5
         mon mgr beacon grace: 90
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/singleton/msgr-failures/few.yaml b/qa/suites/rados/singleton/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/singleton/msgr-failures/many.yaml b/qa/suites/rados/singleton/msgr-failures/many.yaml
@@ -5,6 +5,7 @@ overrides:
         ms inject socket failures: 1000
         mon mgr beacon grace: 90
         mon client hunt interval max multiple: 2
+        mon client directed command retry: 5
       mgr:
         debug monc: 10
     log-whitelist:

diff --git a/qa/suites/rados/thrash-old-clients/msgr-failures/fastclose.yaml b/qa/suites/rados/thrash-old-clients/msgr-failures/fastclose.yaml
@@ -4,5 +4,6 @@ overrides:
       global:
         ms inject socket failures: 2500
         ms tcp read timeout: 5
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/thrash-old-clients/msgr-failures/few.yaml b/qa/suites/rados/thrash-old-clients/msgr-failures/few.yaml
@@ -3,6 +3,7 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
       osd:
         osd heartbeat use min delay socket: true
     log-whitelist:

diff --git a/qa/suites/rados/thrash-old-clients/msgr-failures/osd-delay.yaml b/qa/suites/rados/thrash-old-clients/msgr-failures/osd-delay.yaml
@@ -7,5 +7,6 @@ overrides:
         ms inject delay probability: .005
         ms inject delay max: 1
         ms inject internal delays: .002
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/thrash/msgr-failures/fastclose.yaml b/qa/suites/rados/thrash/msgr-failures/fastclose.yaml
@@ -4,5 +4,6 @@ overrides:
       global:
         ms inject socket failures: 2500
         ms tcp read timeout: 5
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/thrash/msgr-failures/few.yaml b/qa/suites/rados/thrash/msgr-failures/few.yaml
@@ -3,6 +3,7 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
       osd:
         osd heartbeat use min delay socket: true
     log-whitelist:

diff --git a/qa/suites/rados/thrash/msgr-failures/osd-delay.yaml b/qa/suites/rados/thrash/msgr-failures/osd-delay.yaml
@@ -7,5 +7,6 @@ overrides:
         ms inject delay probability: .005
         ms inject delay max: 1
         ms inject internal delays: .002
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rados/verify/msgr-failures/few.yaml b/qa/suites/rados/verify/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rbd/basic/msgr-failures/few.yaml b/qa/suites/rbd/basic/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rbd/cli/msgr-failures/few.yaml b/qa/suites/rbd/cli/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rbd/cli_v1/msgr-failures/few.yaml b/qa/suites/rbd/cli_v1/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+	mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rbd/librbd/msgr-failures/few.yaml b/qa/suites/rbd/librbd/msgr-failures/few.yaml
@@ -3,5 +3,7 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
-    - but it is still running
+      - but it is still running
+      - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rbd/qemu/msgr-failures/few.yaml b/qa/suites/rbd/qemu/msgr-failures/few.yaml
@@ -3,6 +3,7 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
     - but it is still running
     - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rbd/thrash/msgr-failures/few.yaml b/qa/suites/rbd/thrash/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/rgw/verify/msgr-failures/few.yaml b/qa/suites/rgw/verify/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/smoke/basic/tasks/mon_thrash.yaml b/qa/suites/smoke/basic/tasks/mon_thrash.yaml
@@ -21,6 +21,7 @@ overrides:
         ms inject delay type: mon
         ms inject internal delays: 0.002
         ms inject socket failures: 2500
+        mon client directed command retry: 5
 tasks:
 - install: null
 - ceph:

diff --git a/qa/suites/smoke/basic/tasks/rados_bench.yaml b/qa/suites/smoke/basic/tasks/rados_bench.yaml
@@ -7,6 +7,7 @@ overrides:
         ms inject delay type: osd
         ms inject internal delays: 0.002
         ms inject socket failures: 2500
+        mon client directed command retry: 5
 tasks:
 - install: null
 - ceph:

diff --git a/qa/suites/smoke/basic/tasks/rbd_fsx.yaml b/qa/suites/smoke/basic/tasks/rbd_fsx.yaml
@@ -17,6 +17,7 @@ overrides:
         rbd cache: true
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
 tasks:
 - install: null
 - ceph:

diff --git a/qa/suites/tgt/basic/msgr-failures/few.yaml b/qa/suites/tgt/basic/msgr-failures/few.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 5000
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/tgt/basic/msgr-failures/many.yaml b/qa/suites/tgt/basic/msgr-failures/many.yaml
@@ -3,5 +3,6 @@ overrides:
     conf:
       global:
         ms inject socket failures: 500
+        mon client directed command retry: 5
     log-whitelist:
       - \(OSD_SLOW_PING_TIME