Add multi-master-no-forward command to reduce bus traffic with multi-master

Author: JohnSully

keydb.conf

@@ -1815,6 +1815,12 @@ jemalloc-bg-thread yes
 #  Tuning this parameter is a tradeoff between locking overhead and distributing the workload over multiple cores
 # min-clients-per-thread 50
 
+# Avoid forwarding RREPLAY messages to other masters?
+#   WARNING: This setting is dangerous! You must be certain all masters are connected to each
+#   other in a true mesh topology or data loss will occur!
+#   This command can be used to reduce multimaster bus traffic
+# multi-master-no-forward no
+
 # Path to directory for file backed scratchpad.  The file backed scratchpad
 # reduces memory requirements by storing rarely accessed data on disk 
 # instead of RAM.  A temporary file will be created in this directory.

src/config.cpp

@@ -2167,6 +2167,13 @@ static int updateMaxclients(long long val, long long prev, const char **err) {
     return 1;
 }
 
+static int validateMultiMasterNoForward(int val, const char **) {
+    if (val) {
+        serverLog(LL_WARNING, "WARNING: multi-master-no-forward is set, you *must* use a mesh topology or dataloss will occur");
+    }
+    return 1;
+}
+
 #ifdef USE_OPENSSL
 static int updateTlsCfg(char *val, char *prev, const char **err) {
     UNUSED(val);
@@ -2222,6 +2229,7 @@ standardConfig configs[] = {
     createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, g_pserver->cluster_enabled, 0, NULL, NULL),
     createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, g_pserver->aof_enabled, 0, NULL, updateAppendonly),
     createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_reads_when_down, 0, NULL, NULL),
+    createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL),
 
     /* String Configs */
     createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL),

src/replication.cpp

@@ -4251,7 +4251,7 @@ void replicaReplayCommand(client *c)
     serverTL->current_client = current_clientSave;
 
     // call() will not propogate this for us, so we do so here
-    if (!s_pstate->FCancelled() && s_pstate->FFirst())
+    if (!s_pstate->FCancelled() && s_pstate->FFirst() && !cserver.multimaster_no_forward)
         alsoPropagate(cserver.rreplayCommand,c->db->id,c->argv,c->argc,PROPAGATE_AOF|PROPAGATE_REPL);
 
     s_pstate->Pop();

src/server.h

@@ -1648,6 +1648,7 @@ struct redisServerConst {
     unsigned char uuid[UUID_BINARY_LEN];         /* This server's UUID - populated on boot */
     bool fUsePro = false;
     int thread_min_client_threshold = 50;
+    int multimaster_no_forward;
 };
 
 struct redisServer {

tests/integration/replication-multimaster.tcl

@@ -1,4 +1,6 @@
 foreach topology {mesh ring} {
+
+foreach noforward [expr {[string equal $topology "mesh"] ? {no yes} : {no}}] {
 start_server {tags {"multi-master"} overrides {hz 500 active-replica yes multi-master yes}} {
 start_server {overrides {hz 500 active-replica yes multi-master yes}} {
 start_server {overrides {hz 500 active-replica yes multi-master yes}} {
@@ -8,8 +10,12 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
         set R($j) [srv [expr 0-$j] client]
         set R_host($j) [srv [expr 0-$j] host]
         set R_port($j) [srv [expr 0-$j] port]
+
+	$R($j) config set multi-master-no-forward $noforward
     }
 
+    set topology_name "$topology[expr {[string equal $noforward "yes"] ? " no-forward" : ""}]"
+
     # Initialize as mesh
     if [string equal $topology "mesh"] {
     for {set j 0} {$j < 4} {incr j} {
@@ -31,7 +37,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
         $R(3) replicaof $R_host(2) $R_port(2)
     }
 
-    test "$topology all nodes up" {
+    test "$topology_name all nodes up" {
         for {set j 0} {$j < 4} {incr j} {
             wait_for_condition 50 100 {
                 [string match {*master_global_link_status:up*} [$R($j) info replication]]
@@ -41,7 +47,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
         }
     }
 
-    test "$topology replicates to all nodes" {
+    test "$topology_name replicates to all nodes" {
         $R(0) set testkey foo
         after 500
         for {set n 0} {$n < 4} {incr n} {
@@ -53,7 +59,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
         }
     }
 
-    test "$topology replicates only once" {
+    test "$topology_name replicates only once" {
         $R(0) set testkey 1
         after 500
         #wait_for_condition 50 100 {
@@ -74,7 +80,7 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
         }
     }
 
-    test "$topology transaction replicates only once" {
+    test "$topology_name transaction replicates only once" {
         for {set j 0} {$j < 1000} {incr j} {
             $R(0) set testkey 1
             $R(0) multi
@@ -95,3 +101,4 @@ start_server {overrides {hz 500 active-replica yes multi-master yes}} {
 }
 }
 }
+}