watchdog: Check I/O every n-seconds

pr-corp-acc · Jul 30, 2019 · 29c684b · 29c684b
1 parent e784588
commit 29c684b
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 66 deletions.
diff --git a/bbinc/cr_utils.h b/bbinc/cr_utils.h
diff --git a/bdb/attr.h b/bdb/attr.h
@@ -670,6 +670,8 @@ DEF_ATTR(
 DEF_ATTR(
     AA_REQUEST_MODE, aa_request_mode, BOOLEAN, 0,
     "Print a message to stdout instead of performing auto-analyze ourselves")
+DEF_ATTR(
+    TEST_IO_TIME, test_io_time, SECS, 10, "Check I/O in watchdog this often")
 
 /*
   BDB_ATTR_REPTIMEOUT

diff --git a/db/watchdog.c b/db/watchdog.c
@@ -165,6 +165,7 @@ static void *watchdog_thread(void *arg)
     while (!gbl_ready)
         sleep(1);
 
+    int test_io_time = 0;
     while (!db_is_stopped()) {
         gbl_epoch_time = comdb2_time_epoch();
 
@@ -316,8 +317,15 @@ static void *watchdog_thread(void *arg)
                to do. */
             bdb_flush_up_to_lsn(thedb->bdb_env, 1, 0);
 
-            if (bdb_watchdog_test_io(thedb->bdb_env))
-                its_bad = 1;
+            if (gbl_epoch_time - test_io_time >
+                bdb_attr_get(thedb->bdb_attr, BDB_ATTR_TEST_IO_TIME)) {
+                if (bdb_watchdog_test_io(thedb->bdb_env)) {
+                    logmsg(LOGMSG_FATAL,
+                           "%s:bdb_watchdog_test_io failed - aborting\n", __func__);
+                    abort();
+                }
+                test_io_time = gbl_epoch_time;
+            }
 
             /* if nothing was bad, update the timestamp */
             if (!its_bad && !its_bad_slow) {