Debug agent to track alive coroutines

  * Can be installed dynamically or from command line
  * Captures coroutine creation stacktrace and stores it in completion, automatically enhancing stacktrace recovery mechanism
  * Allows to dump and introspect all active coroutines
  * Allows to dump Job hierarchy
  * When installed from command line, dumps all coroutines on kill -5
  * Probe support in undispatched coroutines

README.md

@@ -31,6 +31,8 @@ GlobalScope.launch {
 * [core](core/README.md) — Kotlin/JVM implementation of common coroutines with additional features:
   * `Dispatchers.IO` dispatcher for blocking coroutines;
   * `Executor.asCoroutineDispatcher()` extension, custom thread pools, and more.
+* [debug](core/README.md) — debug utilities for coroutines.
+  * `DebugProbes` API to probe, keep track of, print and dump active coroutines.
 * [js](js/README.md) — Kotlin/JS implementation of common coroutines with `Promise` support.
 * [native](native/README.md) — Kotlin/Native implementation of common coroutines with `runBlocking` single-threaded event loop.
 * [reactive](reactive/README.md) — modules that provide builders and iteration support for various reactive streams libraries:

RELEASE.md

@@ -12,7 +12,7 @@ To release new `<version>` of `kotlinx-coroutines`:
    `git merge origin/master`   
 
 4. Search & replace `<old-version>` with `<version>` across the project files. Should replace in:
-   * [`README.md`](README.md)
+   * [`README.md`](README.md) (native, core, test, debug, modules)
    * [`coroutines-guide.md`](docs/coroutines-guide.md)
    * [`gradle.properties`](gradle.properties)
    * [`ui/kotlinx-coroutines-android/example-app/gradle.properties`](ui/kotlinx-coroutines-android/example-app/gradle.properties)    

binary-compatibility-validator/build.gradle

@@ -13,6 +13,7 @@ dependencies {
     testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version"
 
     testArtifacts project(':kotlinx-coroutines-core')
+    testArtifacts project(':kotlinx-coroutines-debug')
 
     testArtifacts project(':kotlinx-coroutines-reactive')
     testArtifacts project(':kotlinx-coroutines-reactor')

binary-compatibility-validator/reference-public-api/kotlinx-coroutines-core.txt

@@ -349,6 +349,7 @@ public class kotlinx/coroutines/JobSupport : kotlinx/coroutines/ChildJob, kotlin
 	public fun plus (Lkotlinx/coroutines/Job;)Lkotlinx/coroutines/Job;
 	public final fun registerSelectClause0 (Lkotlinx/coroutines/selects/SelectInstance;Lkotlin/jvm/functions/Function1;)V
 	public final fun start ()Z
+	public final fun toDebugString ()Ljava/lang/String;
 	public fun toString ()Ljava/lang/String;
 }
 

binary-compatibility-validator/reference-public-api/kotlinx-coroutines-debug.txt

@@ -0,0 +1,39 @@
+public final class kotlinx/coroutines/debug/CoroutineState {
+	public final fun component1 ()Lkotlin/coroutines/Continuation;
+	public final fun component2 ()Ljava/util/List;
+	public final fun copy (Lkotlin/coroutines/Continuation;Ljava/util/List;J)Lkotlinx/coroutines/debug/CoroutineState;
+	public static synthetic fun copy$default (Lkotlinx/coroutines/debug/CoroutineState;Lkotlin/coroutines/Continuation;Ljava/util/List;JILjava/lang/Object;)Lkotlinx/coroutines/debug/CoroutineState;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getContinuation ()Lkotlin/coroutines/Continuation;
+	public final fun getCreationStackTrace ()Ljava/util/List;
+	public final fun getJob ()Lkotlinx/coroutines/Job;
+	public final fun getJobOrNull ()Lkotlinx/coroutines/Job;
+	public final fun getState ()Lkotlinx/coroutines/debug/State;
+	public fun hashCode ()I
+	public final fun lastObservedStackTrace ()Ljava/util/List;
+	public fun toString ()Ljava/lang/String;
+}
+
+public final class kotlinx/coroutines/debug/DebugProbes {
+	public static final field INSTANCE Lkotlinx/coroutines/debug/DebugProbes;
+	public final fun dumpCoroutines (Ljava/io/PrintStream;)V
+	public static synthetic fun dumpCoroutines$default (Lkotlinx/coroutines/debug/DebugProbes;Ljava/io/PrintStream;ILjava/lang/Object;)V
+	public final fun dumpCoroutinesState ()Ljava/util/List;
+	public final fun getSanitizeStackTraces ()Z
+	public final fun hierarchyToString (Lkotlinx/coroutines/Job;)Ljava/lang/String;
+	public final fun install ()V
+	public final fun printHierarchy (Lkotlinx/coroutines/Job;Ljava/io/PrintStream;)V
+	public static synthetic fun printHierarchy$default (Lkotlinx/coroutines/debug/DebugProbes;Lkotlinx/coroutines/Job;Ljava/io/PrintStream;ILjava/lang/Object;)V
+	public final fun setSanitizeStackTraces (Z)V
+	public final fun uninstall ()V
+	public final fun withDebugProbes (Lkotlin/jvm/functions/Function0;)V
+}
+
+public final class kotlinx/coroutines/debug/State : java/lang/Enum {
+	public static final field CREATED Lkotlinx/coroutines/debug/State;
+	public static final field RUNNING Lkotlinx/coroutines/debug/State;
+	public static final field SUSPENDED Lkotlinx/coroutines/debug/State;
+	public static fun valueOf (Ljava/lang/String;)Lkotlinx/coroutines/debug/State;
+	public static fun values ()[Lkotlinx/coroutines/debug/State;
+}
+

common/kotlinx-coroutines-core-common/src/JobSupport.kt

@@ -925,7 +925,10 @@ public open class JobSupport constructor(active: Boolean) : Job, ChildJob, Paren
 
     // for nicer debugging
     public override fun toString(): String =
-        "${nameString()}{${stateString(state)}}@$hexAddress"
+        "${toDebugString()}@$hexAddress"
+
+    @InternalCoroutinesApi
+    public fun toDebugString(): String = "${nameString()}{${stateString(state)}}"
 
     /**
      * @suppress **This is unstable API and it is subject to change.**

common/kotlinx-coroutines-core-common/src/internal/ProbesSupport.common.kt

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.coroutines.internal
+
+import kotlin.coroutines.*
+
+internal expect inline fun <T> probeCoroutineCreated(completion: Continuation<T>): Continuation<T>

common/kotlinx-coroutines-core-common/src/intrinsics/Undispatched.kt

@@ -15,8 +15,8 @@ import kotlin.coroutines.intrinsics.*
  * It does not use [ContinuationInterceptor] and does not update context of the current thread.
  */
 internal fun <T> (suspend () -> T).startCoroutineUnintercepted(completion: Continuation<T>) {
-    startDirect(completion) {
-        startCoroutineUninterceptedOrReturn(completion)
+    startDirect(completion) { actualCompletion ->
+        startCoroutineUninterceptedOrReturn(actualCompletion)
     }
 }
 
@@ -26,8 +26,8 @@ internal fun <T> (suspend () -> T).startCoroutineUnintercepted(completion: Conti
  * It does not use [ContinuationInterceptor] and does not update context of the current thread.
  */
 internal fun <R, T> (suspend (R) -> T).startCoroutineUnintercepted(receiver: R, completion: Continuation<T>) {
-    startDirect(completion) {
-        startCoroutineUninterceptedOrReturn(receiver, completion)
+    startDirect(completion) {  actualCompletion ->
+        startCoroutineUninterceptedOrReturn(receiver, actualCompletion)
     }
 }
 
@@ -37,9 +37,9 @@ internal fun <R, T> (suspend (R) -> T).startCoroutineUnintercepted(receiver: R,
  * It does not use [ContinuationInterceptor], but updates the context of the current thread for the new coroutine.
  */
 internal fun <T> (suspend () -> T).startCoroutineUndispatched(completion: Continuation<T>) {
-    startDirect(completion) {
+    startDirect(completion) { actualCompletion ->
         withCoroutineContext(completion.context, null) {
-            startCoroutineUninterceptedOrReturn(completion)
+            startCoroutineUninterceptedOrReturn(actualCompletion)
         }
     }
 }
@@ -50,23 +50,29 @@ internal fun <T> (suspend () -> T).startCoroutineUndispatched(completion: Contin
  * It does not use [ContinuationInterceptor], but updates the context of the current thread for the new coroutine.
  */
 internal fun <R, T> (suspend (R) -> T).startCoroutineUndispatched(receiver: R, completion: Continuation<T>) {
-    startDirect(completion) {
+    startDirect(completion) { actualCompletion ->
         withCoroutineContext(completion.context, null) {
-            startCoroutineUninterceptedOrReturn(receiver, completion)
+            startCoroutineUninterceptedOrReturn(receiver, actualCompletion)
         }
     }
 }
 
-private inline fun <T> startDirect(completion: Continuation<T>, block: () -> Any?) {
+/**
+ * Starts given [block] immediately in the current stack-frame until first suspension point.
+ * This method supports debug probes and thus can intercept completion, thus completion is provide
+ * as the parameter of [block].
+ */
+private inline fun <T> startDirect(completion: Continuation<T>, block: (Continuation<T>) -> Any?) {
+    val actualCompletion = probeCoroutineCreated(completion)
     val value = try {
-        block()
+        block(actualCompletion)
     } catch (e: Throwable) {
-        completion.resumeWithException(e)
+        actualCompletion.resumeWithException(e)
         return
     }
     if (value !== COROUTINE_SUSPENDED) {
         @Suppress("UNCHECKED_CAST")
-        completion.resume(value as T)
+        actualCompletion.resume(value as T)
     }
 }
 

core/README.md

@@ -5,5 +5,5 @@ Module name below corresponds to the artifact name in Maven/Gradle.
 
 ## Modules
 
-* [kotlinx-coroutines-core](kotlinx-coroutines-core/README.md) -- core coroutine builders and synchronization primitives. 
-
+* [kotlinx-coroutines-core](kotlinx-coroutines-core/README.md) — core coroutine builders and synchronization primitives. 
+* [kotlinx-coroutines-debug](kotlinx-coroutines-debug/README.md) — coroutines debug utilities.

core/kotlinx-coroutines-core/src/Debug.kt

@@ -41,6 +41,7 @@ public const val DEBUG_PROPERTY_VALUE_ON = "on"
  */
 public const val DEBUG_PROPERTY_VALUE_OFF = "off"
 
+@JvmField
 internal val DEBUG = systemProp(DEBUG_PROPERTY_NAME).let { value ->
     when (value) {
         DEBUG_PROPERTY_VALUE_AUTO, null -> CoroutineId::class.java.desiredAssertionStatus()
@@ -50,7 +51,8 @@ internal val DEBUG = systemProp(DEBUG_PROPERTY_NAME).let { value ->
     }
 }
 
-internal val RECOVER_STACKTRACE = systemProp(STACKTRACE_RECOVERY_PROPERTY_NAME, true)
+@JvmField
+internal val RECOVER_STACKTRACES = systemProp(STACKTRACE_RECOVERY_PROPERTY_NAME, true)
 
 // internal debugging tools
 

core/kotlinx-coroutines-core/src/internal/ProbesSupport.kt

@@ -0,0 +1,11 @@
+/*
+ * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+@file:Suppress("NOTHING_TO_INLINE", "INVISIBLE_REFERENCE", "INVISIBLE_MEMBER")
+
+package kotlinx.coroutines.internal
+
+import kotlin.coroutines.*
+import kotlin.coroutines.jvm.internal.probeCoroutineCreated as probe
+
+internal actual inline fun <T> probeCoroutineCreated(completion: Continuation<T>): Continuation<T> = probe(completion)

core/kotlinx-coroutines-core/src/internal/StackTraceRecovery.kt

@@ -165,7 +165,7 @@ internal actual fun <E : Throwable> unwrap(exception: E): E {
 }
 
 private fun <E : Throwable> recoveryDisabled(exception: E) =
-    !RECOVER_STACKTRACE || !DEBUG || exception is CancellationException || exception is NonRecoverableThrowable
+    !RECOVER_STACKTRACES || !DEBUG || exception is CancellationException || exception is NonRecoverableThrowable
 
 private fun createStackTrace(continuation: CoroutineStackFrame): ArrayDeque<StackTraceElement> {
     val stack = ArrayDeque<StackTraceElement>()
@@ -179,14 +179,17 @@ private fun createStackTrace(continuation: CoroutineStackFrame): ArrayDeque<Stac
     return stack
 }
 
-internal fun sanitize(element: StackTraceElement): StackTraceElement {
+@InternalCoroutinesApi
+public fun sanitize(element: StackTraceElement): StackTraceElement {
     if (!element.className.contains('/')) {
         return element
     }
     // KT-28237: STE generated with debug metadata contains '/' as separators in FQN, while Java contains dots
     return StackTraceElement(element.className.replace('/', '.'), element.methodName, element.fileName, element.lineNumber)
 }
-internal fun artificialFrame(message: String) = java.lang.StackTraceElement("\b\b\b($message", "\b", "\b", -1)
+
+@InternalCoroutinesApi
+public fun artificialFrame(message: String) = java.lang.StackTraceElement("\b\b\b($message", "\b", "\b", -1)
 internal fun StackTraceElement.isArtificial() = className.startsWith("\b\b\b")
 private fun Array<StackTraceElement>.frameIndex(methodName: String) = indexOfFirst { methodName == it.className }
 

core/kotlinx-coroutines-debug/README.md

@@ -0,0 +1,118 @@
+# Module kotlinx-coroutines-debug
+
+Debugging facilities for `kotlinx.coroutines` on JVM.
+
+### Overview
+This module provides a debug JVM agent which allows to track and trace alive coroutines.
+Main entry point to debug facilities is [DebugProbes].
+Call to [DebugProbes.install] installs debug agent via ByteBuddy and starts to spy on coroutines when they are created, suspended or resumed.
+
+After that you can use [DebugProbes.dumpCoroutines] to print all active (suspended or running) coroutines, including their state, creation and
+suspension stacktraces.
+Additionally, it is possible to process list of such coroutines via [DebugProbes.dumpCoroutinesState] or dump isolated parts
+of coroutines hierarchies referenced by [Job] instance using [DebugProbes.printHierarchy].
+
+### Using as JVM agent
+Additionally, it is possible to use this module as standalone JVM agent to enable debug probes on the application startup.
+You can run your application with additional argument: `-javaagent:kotlinx-coroutines-debug-1.1.0.jar`.
+Additionally, on Linux and Mac OS X you can use `kill -5 $pid` command in order to force your application to print all alive coroutines.
+
+
+### Example of usage
+
+Capabilities of this module can be demonstrated by the following example:
+```kotlin
+class Computation {
+    public fun computeValue(): Deferred<String> = GlobalScope.async {
+        val firstPart = computeFirstPart()
+        val secondPart = computeSecondPart()
+
+        combineResults(firstPart, secondPart)
+    }
+
+    private suspend fun combineResults(firstPart: Deferred<String>, secondPart: Deferred<String>): String {
+        return firstPart.await() + secondPart.await()
+    }
+
+
+    private suspend fun CoroutineScope.computeFirstPart() = async {
+        delay(5000)
+        "4"
+    }
+
+    private suspend fun CoroutineScope.computeSecondPart() = async {
+        delay(5000)
+        "2"
+    }
+}
+
+fun main(args: Array<String>) = runBlocking {
+    DebugProbes.install()
+    val computation = Computation()
+    val deferred = computation.computeValue()
+
+    // Delay for some time
+    delay(1000)
+
+    DebugProbes.dumpCoroutines()
+
+    println("\nDumping only deferred")
+    DebugProbes.printHierarchy(deferred)
+}
+```
+
+Printed result will be:
+```
+Coroutines dump 2018/11/12 21:44:02
+
+Coroutine "coroutine#2":DeferredCoroutine{Active}@1b26f7b2, state: SUSPENDED
+	at kotlinx.coroutines.DeferredCoroutine.await$suspendImpl(Builders.common.kt:99)
+	at Computation.combineResults(Example.kt:18)
+	at Computation$computeValue$1.invokeSuspend(Example.kt:14)
+	(Coroutine creation stacktrace)
+	at kotlin.coroutines.intrinsics.IntrinsicsKt__IntrinsicsJvmKt.createCoroutineUnintercepted(IntrinsicsJvm.kt:116)
+	at kotlinx.coroutines.intrinsics.CancellableKt.startCoroutineCancellable(Cancellable.kt:23)
+	at kotlinx.coroutines.CoroutineStart.invoke(CoroutineStart.kt:109)
+	at kotlinx.coroutines.AbstractCoroutine.start(AbstractCoroutine.kt:160)
+	at kotlinx.coroutines.BuildersKt__Builders_commonKt.async(Builders.common.kt:88)
+	at kotlinx.coroutines.BuildersKt.async(Unknown Source)
+	at kotlinx.coroutines.BuildersKt__Builders_commonKt.async$default(Builders.common.kt:81)
+	at kotlinx.coroutines.BuildersKt.async$default(Unknown Source)
+	at Computation.computeValue(Example.kt:10)
+	at ExampleKt$main$1.invokeSuspend(Example.kt:36)
+	at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:32)
+	at kotlinx.coroutines.DispatchedTask$DefaultImpls.run(Dispatched.kt:237)
+	at kotlinx.coroutines.DispatchedContinuation.run(Dispatched.kt:81)
+	at kotlinx.coroutines.EventLoopBase.processNextEvent(EventLoop.kt:123)
+	at kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:69)
+	at kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:45)
+	at kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source)
+	at kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking$default(Builders.kt:35)
+	at kotlinx.coroutines.BuildersKt.runBlocking$default(Unknown Source)
+	at ExampleKt.main(Example.kt:33)
+
+... More coroutines here ...
+
+Dumping only deferred
+"coroutine#2":DeferredCoroutine{Active}, continuation is SUSPENDED at line kotlinx.coroutines.DeferredCoroutine.await$suspendImpl(Builders.common.kt:99)
+	"coroutine#3":DeferredCoroutine{Active}, continuation is SUSPENDED at line Computation$computeFirstPart$2.invokeSuspend(Example.kt:23)
+	"coroutine#4":DeferredCoroutine{Active}, continuation is SUSPENDED at line Computation$computeSecondPart$2.invokeSuspend(Example.kt:28)
+```
+
+
+### Status of the API
+
+API is purely experimental and it is not guaranteed that it won't be changed (while it is marked as `@ExperimentalCoroutinesApi`).
+Do not use this module in production environment and do not rely on the format of the data produced by [DebugProbes]. 
+
+<!--- MODULE kotlinx-coroutines-core -->
+<!--- INDEX kotlinx.coroutines -->
+[Job]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-core/kotlinx.coroutines/-job/index.html
+<!--- MODULE kotlinx-coroutines-debug -->
+<!--- INDEX kotlinx.coroutines.debug -->
+[DebugProbes]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/index.html
+[DebugProbes.install]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/install.html
+[DebugProbes.dumpCoroutines]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/dump-coroutines.html
+[DebugProbes.dumpCoroutinesState]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/dump-coroutines-state.html
+[DebugProbes.printHierarchy]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/print-hierarchy.html
+<!--- END -->

core/kotlinx-coroutines-debug/build.gradle

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+dependencies {
+    compile "net.bytebuddy:byte-buddy:$byte_buddy_version"
+    compile "net.bytebuddy:byte-buddy-agent:$byte_buddy_version"
+}
+
+jar {
+    manifest {
+        attributes "Premain-Class": "kotlinx.coroutines.debug.AgentPremain"
+        attributes "Can-Redefine-Classes": "true"
+        // For local runs
+        // attributes "Main-Class": "kotlinx.coroutines.debug.Playground"
+        // attributes "Class-Path": configurations.compile.collect { it.absolutePath }.join(" ")
+    }
+}

core/kotlinx-coroutines-debug/src/debug/AgentPremain.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.coroutines.debug
+
+import net.bytebuddy.agent.*
+import sun.misc.*
+import java.lang.instrument.*
+
+@Suppress("unused")
+internal object AgentPremain {
+
+    @JvmStatic
+    public fun premain(args: String?, instrumentation: Instrumentation) {
+        Installer.premain(args, instrumentation)
+        DebugProbes.install()
+        installSignalHandler()
+    }
+
+    private fun installSignalHandler() {
+        val signal = Signal("TRAP") // kill -5
+        Signal.handle(signal, { DebugProbes.dumpCoroutines() })
+    }
+}