Fix Fray hang when a thread exits but the monitor lock is acquired by another thread. (#151)

Author: aoli-al

CHANGELOG.md

@@ -12,6 +12,8 @@
 
 ### Fixed
 
+- Fix Fray hang when a thread exits but the monitor lock is not released.
+
 ### Security
 
 ## 0.3.0 - 2025-03-10

core/src/main/kotlin/org/pastalab/fray/core/RunContext.kt

@@ -337,7 +337,7 @@ class RunContext(val config: Configuration) {
   fun threadCompleted(t: Thread) {
     val context = registeredThreads[t.id]!!
     context.isExiting = true
-    monitorEnter(t)
+    monitorEnter(t, true)
     objectNotifyAll(t)
     context.state = ThreadState.Completed
     //    lockManager.threadUnblockedDueToDeadlock(t)
@@ -539,15 +539,16 @@ class RunContext(val config: Configuration) {
   }
 
   fun lockTryLock(lock: Any, canInterrupt: Boolean, timed: Boolean) {
-    lockImpl(lock, false, false, canInterrupt, timed)
+    lockImpl(lock, false, false, canInterrupt, timed, false)
   }
 
   fun lockImpl(
       lock: Any,
       isMonitorLock: Boolean,
       shouldBlock: Boolean,
       canInterrupt: Boolean,
-      timed: Boolean
+      timed: Boolean,
+      shouldRetry: Boolean,
   ) {
     val t = Thread.currentThread().id
     val context = registeredThreads[t]!!
@@ -587,7 +588,11 @@ class RunContext(val config: Configuration) {
       // not want to rely on ReentrantLock. This allows
       // us to pick which Thread to run next if multiple
       // threads hold the same lock.
-      scheduleNextOperation(true)
+      if (shouldRetry) {
+        scheduleNextOperationAndCheckDeadlock(true)
+      } else {
+        scheduleNextOperation(true)
+      }
       if (canInterrupt) {
         context.checkInterrupt()
       }
@@ -599,12 +604,12 @@ class RunContext(val config: Configuration) {
     }
   }
 
-  fun monitorEnter(lock: Any) {
-    lockImpl(lock, true, true, false, false)
+  fun monitorEnter(lock: Any, shouldRetry: Boolean) {
+    lockImpl(lock, true, true, false, false, shouldRetry)
   }
 
   fun lockLock(lock: Any, canInterrupt: Boolean) {
-    lockImpl(lock, false, true, canInterrupt, false)
+    lockImpl(lock, false, true, canInterrupt, false, false)
   }
 
   fun reentrantReadWriteLockInit(
@@ -1028,18 +1033,9 @@ class RunContext(val config: Configuration) {
 
     // The second empty check throws deadlock exceptions.
     if (enabledOperations.isEmpty()) {
-      if (registeredThreads.all { it.value.state == ThreadState.Completed }) {
-        // We are done here, we should go back to the main thread.
-        if (currentThreadId != mainThreadId) {
-          registeredThreads[mainThreadId]!!.unblock()
-        }
-        return
-      } else if (!currentThread.isExiting || Thread.currentThread() is HelperThread) {
-        // Deadlock detected
-        val e = org.pastalab.fray.runtime.DeadlockException()
-        reportError(e)
-        throw e
-      }
+      val e = DeadlockException()
+      reportError(e)
+      throw e
     }
 
     step += 1

core/src/main/kotlin/org/pastalab/fray/core/RuntimeDelegate.kt

@@ -239,7 +239,7 @@ class RuntimeDelegate(val context: RunContext) : org.pastalab.fray.runtime.Deleg
   override fun onMonitorEnter(o: Any) {
     if (checkEntered()) return
     try {
-      context.monitorEnter(o)
+      context.monitorEnter(o, false)
     } finally {
       entered.set(false)
     }

integration-test/src/main/java/org/pastalab/fray/test/fail/thread/ThreadExitDeadlock.java

@@ -0,0 +1,15 @@
+package org.pastalab.fray.test.fail.thread;
+
+import java.util.concurrent.CountDownLatch;
+
+public class ThreadExitDeadlock {
+    public static void main(String[] args) throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(1);
+        Thread t = new Thread(() -> {
+        });
+        synchronized (t) {
+            t.start();
+            latch.await();
+        }
+    }
+}

integration-test/src/test/java/org/pastalab/fray/test/FrayTestCase.java

@@ -17,6 +17,7 @@
 import org.pastalab.fray.test.fail.cdl.CountDownLatchDeadlockUnblockMultiThread;
 import org.pastalab.fray.test.fail.park.ParkDeadlock;
 import org.pastalab.fray.test.fail.rwlock.ReentrantReadWriteLockDeadlock;
+import org.pastalab.fray.test.fail.thread.ThreadExitDeadlock;
 import org.pastalab.fray.test.fail.wait.NotifyOrder;
 import org.pastalab.fray.test.success.condition.ConditionAwaitTimeoutNotifyInterrupt;
 import org.pastalab.fray.test.success.rwlock.ReentrantReadWriteLockDowngradingNoDeadlock;
@@ -77,7 +78,7 @@ public void testOne() throws Throwable {
                     new ExecutionInfo(
                             new LambdaExecutor(() -> {
                                 try {
-                                    NotifyOrder.main(new String[]{});
+                                    ThreadExitDeadlock.main(new String[]{});
                                 } catch (Exception e) {
                                     throw new RuntimeException(e);
                                 }
@@ -90,7 +91,7 @@ public void testOne() throws Throwable {
                     "/tmp/report2",
                     1000,
                     60,
-                    new RandomScheduler(),
+                    new POSScheduler(),
                     new ControlledRandom(),
                     true,
                     false,