YugaByte DB: retry some DDL errors on client setup

yugabyte/src/yugabyte/ysql/append.clj

@@ -72,7 +72,8 @@
                 (c/execute! c (j/create-table-ddl table
                                                   [;[:k :int "unique"]
                                                    [:k :int "PRIMARY KEY"]
-                                                   [:v :text]]))))
+                                                   [:v :text]]
+                                                  {:conditional? true}))))
          dorun))
 
   (invoke-op! [this test op c conn-wrapper]

yugabyte/src/yugabyte/ysql/client.clj

@@ -280,6 +280,22 @@
                                    {:type :conn-not-ready})))
                  ~@body))
 
+(defmacro with-ddl-retry
+  "YB loves to throw all kinds of exceptions for DDL operations, like
+  encountering OID conflicts when creating multiple tables in a row, or
+  violating uniqueness constraints. This macro catches those and retries them a
+  few times, to raise our chances of running a test successfully."
+  [& body]
+  `(util/with-retry [attempts# max-retry-attempts]
+     ~@body
+     (catch org.postgresql.util.PSQLException e#
+       (let [m# (.getMessage e#)]
+         (if (or (re-find #"duplicate key value violates unique constraint" m#))
+           (do (info "Caught" m# "during DDL setup; retrying.")
+               (Thread/sleep (rand-int max-delay-between-retries-ms))
+               (~'retry (dec attempts#)))
+           (throw e#))))))
+
 (defn with-idempotent
   "Takes a predicate on operation functions, and an op, presumably resulting
   from a client call. If (idempotent? (:f op)) is truthy, remaps :info types to
@@ -373,12 +389,13 @@
     (->YSQLMyClient arg1 arg2 arg3)"
   [class-name inner-client-record]
 
-  ; Since this macro is insanely complex, I'll try to thoroughly comment what's happening here.
-  ; Good luck.
+  ; Since this macro is insanely complex, I'll try to thoroughly comment what's
+  ; happening here. Good luck.
 
-  ; First, before we start with the actual macro output, we analyze inner-client-record constructor
-  ; in order to get a grip on its arguments list.
-  ; For that we're doing a bunch of string manipulations with the inner-client-record's symbol name.
+  ; First, before we start with the actual macro output, we analyze
+  ; inner-client-record constructor in order to get a grip on its arguments
+  ; list. For that we're doing a bunch of string manipulations with the
+  ; inner-client-record's symbol name.
 
   (let [inner-ctor-ns-prefix (if (qualified-symbol? inner-client-record)
                                (str (namespace inner-client-record) "/")
@@ -388,9 +405,9 @@
         inner-ctor-args-vec  (first (:arglists inner-ctor-meta))]
 
     ; Now we're getting to the output.
-    ; We define a record with a given name extending jepsen.client/Client, which takes an
-    ; instance of inner-client (among other things) and delegates logic to it, wrapping it into
-    ; helper methods.
+    ; We define a record with a given name extending jepsen.client/Client,
+    ; which takes an instance of inner-client (among other things) and
+    ; delegates logic to it, wrapping it into helper methods.
 
     `(do (defrecord ~class-name [~'conn-wrapper ~'inner-client ~'setup? ~'teardown?]
            client/Client
@@ -402,9 +419,10 @@
              (once-per-cluster
                ~'setup?
                (info "Running setup")
-               (with-conn
-                 [~'c ~'conn-wrapper]
-                 (setup-cluster! ~'inner-client ~'test ~'c ~'conn-wrapper))
+               (with-ddl-retry
+                 (with-conn
+                   [~'c ~'conn-wrapper]
+                   (setup-cluster! ~'inner-client ~'test ~'c ~'conn-wrapper)))
                (info "Setup sucessful")))
 
            (invoke! [~'this ~'test ~'op]