chore(debugging): log out details of weird intermittent error (spinna…

…ker#2305)
ravikiranaita · Jun 29, 2018 · 3d20f1a · 3d20f1a
1 parent 768e9c4
commit 3d20f1a
Showing 1 changed file with 22 additions and 19 deletions.
diff --git a/orca-queue/src/main/kotlin/com/netflix/spinnaker/orca/q/handler/CompleteStageHandler.kt b/orca-queue/src/main/kotlin/com/netflix/spinnaker/orca/q/handler/CompleteStageHandler.kt
@@ -90,7 +90,7 @@ class CompleteStageHandler(
           stage.status = status
           stage.endTime = clock.millis()
         } catch (e: Exception) {
-          log.error("Failed to construct after stages", e)
+          log.error("Failed to construct after stages for $stage.id", e)
           stage.status = TERMINAL
           stage.endTime = clock.millis()
         }
@@ -215,25 +215,28 @@ class CompleteStageHandler(
         repository.removeStage(execution, stage.id)
       }
   }
+
+  private fun Stage.determineStatus(): ExecutionStatus {
+    val syntheticStatuses = syntheticStages().map(Stage::getStatus)
+    val taskStatuses = tasks.map(Task::getStatus)
+    val planningStatus = if (hasPlanningFailure()) listOf(failureStatus()) else emptyList()
+    val allStatuses = syntheticStatuses + taskStatuses + planningStatus
+    val afterStageStatuses = afterStages().map(Stage::getStatus)
+    return when {
+      allStatuses.isEmpty()                    -> NOT_STARTED
+      allStatuses.contains(TERMINAL)           -> TERMINAL
+      allStatuses.contains(STOPPED)            -> STOPPED
+      allStatuses.contains(CANCELED)           -> CANCELED
+      allStatuses.contains(FAILED_CONTINUE)    -> FAILED_CONTINUE
+      allStatuses.all { it == SUCCEEDED }      -> SUCCEEDED
+      afterStageStatuses.contains(NOT_STARTED) -> RUNNING // after stages were planned but not run yet
+      else                                     -> {
+        log.error("Unhandled condition for stage $id of $execution.id, marking as TERMINAL. syntheticStatuses=$syntheticStatuses, taskStatuses=$taskStatuses, planningStatus=$planningStatus, afterStageStatuses=$afterStageStatuses")
+        TERMINAL
+      }
+    }
+  }
 }
 
 private fun Stage.hasPlanningFailure() =
   context["beforeStagePlanningFailed"] == true
-
-private fun Stage.determineStatus(): ExecutionStatus {
-  val syntheticStatuses = syntheticStages().map(Stage::getStatus)
-  val taskStatuses = tasks.map(Task::getStatus)
-  val planningStatus = if (hasPlanningFailure()) listOf(failureStatus()) else emptyList()
-  val allStatuses = syntheticStatuses + taskStatuses + planningStatus
-  val afterStageStatuses = afterStages().map(Stage::getStatus)
-  return when {
-    allStatuses.isEmpty()                    -> NOT_STARTED
-    allStatuses.contains(TERMINAL)           -> TERMINAL
-    allStatuses.contains(STOPPED)            -> STOPPED
-    allStatuses.contains(CANCELED)           -> CANCELED
-    allStatuses.contains(FAILED_CONTINUE)    -> FAILED_CONTINUE
-    allStatuses.all { it == SUCCEEDED }      -> SUCCEEDED
-    afterStageStatuses.contains(NOT_STARTED) -> RUNNING // after stages were planned but not run yet
-    else                                     -> TERMINAL
-  }
-}