Improve reproducibility of set test protocol v0 data loss

src/jepsen/mongodb/core.clj

@@ -212,14 +212,13 @@
                        (catch Throwable t
                          (.close conn)
                          (throw t))))
+                   (catch com.mongodb.MongoTimeoutException e
+                     (info "Mongo timeout while waiting for conn; retrying. "
+                           (.getMessage e))
+                     nil)
                    (catch com.mongodb.MongoSocketReadTimeoutException e
                      (info "Mongo socket read timeout waiting for conn; retrying")
-                     nil)
-;                   (catch com.mongodb.MongoServerSelectionException e
-;                     nil))
-                    ; Todo: figure out what Mongo 3.x throws when servers
-                    ; aren't ready yet
-                    )
+                     nil))
                  ; If we aren't ready, sleep and retry
                  (do
                    (Thread/sleep 1000)
@@ -259,9 +258,9 @@
   (assert (integer? (:protocol-version test)))
   {:_id "jepsen"
    :protocolVersion (:protocol-version test)
-   :settings {:heartbeatIntervalMillis 5000  ; protocol v1, ms
-              :electionTimeoutMillis   10000 ; protocol v1, ms
-              :heartbeatTimeoutSecs    10}  ; protocol v0, s
+   :settings {:heartbeatIntervalMillis 2500  ; protocol v1, ms
+              :electionTimeoutMillis   5000 ; protocol v1, ms
+              :heartbeatTimeoutSecs    5}  ; protocol v0, s
    :members (->> test
                  :nodes
                  (map-indexed (fn [i node]
@@ -479,12 +478,12 @@
                                  (info node "mongod killed")))
                              conns))
 
-          :recover (do (nt/reset-time! test)
-                       (info "Clocks reset")
-                       (net/heal! (:net test) test)
-                       (info "Network healed")
-                       (c/on-nodes test start!)
-                       (info "Nodes restarted")))))
+          :stop (do (nt/reset-time! test)
+                    (info "Clocks reset")
+                    (net/heal! (:net test) test)
+                    (info "Network healed")
+                    (c/on-nodes test start!)
+                    (info "Nodes restarted")))))
 
     (teardown! [this test]
                (doseq [[node c] conns]
@@ -496,7 +495,7 @@
     (cycle [{:type :info, :f :isolate, :value nil}
             (gen/sleep 30)
             {:type :info, :f :kill,    :value nil}
-            {:type :info, :f :recover, :value nil}
+            {:type :info, :f :stop,    :value nil}
             (gen/sleep 30)])))
 
 (defn test-
@@ -514,9 +513,16 @@
                                  " p:" (:protocol-version opts))
            :os              debian/os
            :db              (db (:tarball opts))
-           :checker         (checker/perf)
            :nemesis         (primary-divergence-nemesis)
-           :generator       (->> (:generator opts)
-                                 (gen/nemesis (primary-divergence-gen))
-                                 (gen/time-limit (:time-limit opts))))
-    (dissoc opts :generator)))
+           :generator       (gen/phases
+                              (->> (:generator opts)
+                                   (gen/nemesis (primary-divergence-gen))
+                                   (gen/time-limit (:time-limit opts)))
+                              (gen/nemesis
+                                (gen/once {:type :info, :f :stop, :value nil}))
+                              (gen/sleep 20)
+                              (gen/clients
+                                (:final-generator opts))))
+    (dissoc opts
+            :generator
+            :final-generator)))

src/jepsen/mongodb/mongo.clj

@@ -160,9 +160,9 @@
 (defn find-all
   "Finds all docs in a collection."
   [^MongoCollection coll]
-  (-> coll
-      (.find)
-      (map document->map)))
+  (->> coll
+       (.find)
+       (map document->map)))
 
 (defn find-one
   "Find a document by ID."

src/jepsen/mongodb/runner.clj

@@ -8,6 +8,7 @@
             [clojure.string :as str]
             [jepsen.mongodb [core :as m]
                             [mongo :as client]
+                            [set :as set]
                             [document-cas :as dc]]
             [jepsen [cli :as jc]
                     [core :as jepsen]]))
@@ -62,7 +63,7 @@
 
                        ; Run test
                        (doseq [i (range (:test-count options))]
-                         (let [test (jepsen/run! (dc/test options))]
+                         (let [test (jepsen/run! (set/test options))]
                            (when-not (:valid? (:results test))
                              (System/exit 1)))))}})
 

src/jepsen/mongodb/set.clj

@@ -0,0 +1,66 @@
+(ns jepsen.mongodb.set
+  "A big ol set test: lots of inserts, followed by a final read."
+  (:refer-clojure :exclude [test])
+  (:require [clojure [pprint :refer :all]
+                     [string :as str]]
+            [clojure.tools.logging :refer [info debug warn]]
+            [jepsen
+             [util :as util :refer [meh timeout]]
+             [client :as client]
+             [checker :as checker]
+             [generator :as gen]]
+            [jepsen.checker.timeline :as timeline]
+            [jepsen.mongodb.core :refer :all]
+            [jepsen.mongodb.mongo :as m]))
+
+(defrecord Client [db-name coll-name client coll]
+  client/Client
+  (setup! [this test node]
+    (let [client (m/client node)
+          coll  (-> client
+                    (m/db db-name)
+                    (m/collection coll-name)
+                    (m/with-read-concern :linearizable)
+                    (m/with-write-concern :majority))]
+      (assoc this :client client, :coll coll)))
+
+  (invoke! [this test op]
+    (with-errors op #{:read}
+      (case (:f op)
+        :add (let [res (m/insert! coll {:value (:value op)})]
+               (assoc op :type :ok))
+        :read (assoc op
+                     :type :ok
+                     :value (->> coll
+                                 m/find-all
+                                 (map :value)
+                                 (into (sorted-set)))))))
+
+  (teardown! [this test]
+    (.close ^java.io.Closeable client)))
+
+(defn client
+  "A set test client"
+  [opts]
+  (Client. "jepsen" "set" nil nil))
+
+(defn test
+  "A set test, which inserts a sequence of integers into a collection, and
+  performs a final read back."
+  [opts]
+  (test-
+    "set"
+    (merge
+      {:client (client opts)
+       :concurrency (count (:nodes opts))
+       :generator (->> (range)
+                       (map (fn [x] {:type :invoke, :f :add, :value x}))
+                       gen/seq
+                       (gen/stagger 1/2))
+       :final-generator (gen/each
+                          (gen/once {:type :invoke, :f :read, :value nil}))
+       :checker (checker/compose
+                  {:set      checker/set
+                   :timeline (timeline/html)
+                   :perf     (checker/perf)})}
+      opts)))