Don't start multimaster tests until all nodes are connected

Author: JohnSully

src/server.cpp

@@ -4640,8 +4640,17 @@ sds genRedisInfoString(const char *section) {
             listIter li;
             listNode *ln;
             listRewind(g_pserver->masters, &li);
+            bool fAllUp = true;
+            while ((ln = listNext(&li))) {
+                redisMaster *mi = (redisMaster*)listNodeValue(ln);
+                fAllUp = fAllUp && mi->repl_state == REPL_STATE_CONNECTED;
+            }
+
+            sdscatprintf(info, "all_master_link_status:%s\r\n",
+                fAllUp ? "up" : "down");
 
             int cmasters = 0;
+            listRewind(g_pserver->masters, &li);
             while ((ln = listNext(&li)))
             {
                 long long slave_repl_offset = 1;

tests/integration/replication-multimaster.tcl

@@ -31,13 +31,21 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
         $R(3) replicaof $R_host(2) $R_port(2)
     }
 
-    after 2000
+    test "$topology all nodes up" {
+        for {set j 0} {$j < 4} {incr j} {
+            wait_for_condition 50 100 {
+                [string match {*all_master_link_status:up*} [$R($j) info replication]]
+            } else {
+                fail "Multimaster group didn't connect up in a reasonable period of time"
+            }
+        }
+    }
 
     test "$topology replicates to all nodes" {
         $R(0) set testkey foo
         after 500
         for {set n 0} {$n < 4} {incr n} {
-            wait_for_condition 50 1000 {
+            wait_for_condition 50 100 {
                 [$R($n) get testkey] == "foo"
             } else {
                 fail "Failed to replicate to $n"
@@ -48,12 +56,17 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
     test "$topology replicates only once" {
         $R(0) set testkey 1
         after 500
+        #wait_for_condition 50 100 {
+        #    [$R(1) get testkey] == 1 && [$R(2) get testkey] == 1
+        #} else {
+        #    fail "Set failed to replicate"
+        #}
         $R(1) incr testkey
         after 500
         $R(2) incr testkey
         after 500
         for {set n 0} {$n < 4} {incr n} {
-            wait_for_condition 50 1000 {
+            wait_for_condition 100 100 {
                 [$R($n) get testkey] == 3
             } else {
                 fail "node $n did not replicate"
@@ -69,7 +82,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
             $R(0) incr testkey
             $R(0) exec
 	    for {set n 0} {$n < 4} {incr n} {
-	        wait_for_condition 50 1000 {
+	        wait_for_condition 50 100 {
                     [$R($n) get testkey] == 3
                 } else {
                     fail "node $n failed to replicate"