use readcommitted isolation level for event fetch

server/src-lib/Hasura/Eventing/EventTrigger.hs

@@ -187,7 +187,17 @@ processEventQueue logger logenv httpMgr pool getSchemaCache eeCtx@EventEngineCtx
   where
     fetchBatchSize = 100
     popEventsBatch = do
-      let run = liftIO . runExceptT . Q.runTx pool (Q.RepeatableRead, Just Q.ReadWrite)
+      {-
+        SELECT FOR UPDATE .. SKIP LOCKED can throw serialization errors in RepeatableRead: https://stackoverflow.com/a/53289263/1911889
+        We can avoid this safely by running it in ReadCommitted as Postgres will recheck the
+        predicate condition if a row is updated concurrently: https://www.postgresql.org/docs/9.5/transaction-iso.html#XACT-READ-COMMITTED
+
+        Every other action on an event_log row (like post-processing, archival, etc) are single writes (no R-W or W-R)
+        so it is safe to perform them in ReadCommitted as well (the writes will then acquire some serial order).
+        Any serial order of updates to a row will lead to an eventually consistent state as the row will have
+        (delivered=t or error=t or archived=t) after a fixed number of tries (assuming it begins with locked='f').
+      -}
+      let run = liftIO . runExceptT . Q.runTx' pool
       run (fetchEvents fetchBatchSize) >>= \case
           Left err -> do
             liftIO $ L.unLogger logger $ EventInternalErr err