Revert [SPARK-14485][CORE] ignore task finished for executor lost

This reverts commit 695dbc8. This change is being reverted because it hurts performance of some jobs, and only helps in a narrow set of cases. For more discussion, refer to the JIRA. Author: Kay Ousterhout <[email protected]> Closes apache#13580 from kayousterhout/revert-SPARK-14485.
leonelfolmer · Jun 10, 2016 · 5c16ad0 · 5c16ad0
1 parent 2c8f40c
commit 5c16ad0
Showing 1 changed file with 1 addition and 13 deletions.
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -352,29 +352,17 @@ private[spark] class TaskSchedulerImpl(
         }
         taskIdToTaskSetManager.get(tid) match {
           case Some(taskSet) =>
-            var executorId: String = null
             if (TaskState.isFinished(state)) {
               taskIdToTaskSetManager.remove(tid)
               taskIdToExecutorId.remove(tid).foreach { execId =>
-                executorId = execId
                 if (executorIdToTaskCount.contains(execId)) {
                   executorIdToTaskCount(execId) -= 1
                 }
               }
             }
             if (state == TaskState.FINISHED) {
               taskSet.removeRunningTask(tid)
-              // In some case, executor has already been removed by driver for heartbeats timeout,
-              // but at sometime, before executor killed by cluster, the task of running on this
-              // executor is finished and return task success state to driver. However, this kinds
-              // of task should be ignored, because the task on this executor is already re-queued
-              // by driver. For more details, can check in SPARK-14485.
-              if (executorId != null && !executorIdToTaskCount.contains(executorId)) {
-                logInfo(s"Ignoring update with state $state for TID $tid because its executor " +
-                  s"has already been removed by driver")
-              } else {
-                taskResultGetter.enqueueSuccessfulTask(taskSet, tid, serializedData)
-              }
+              taskResultGetter.enqueueSuccessfulTask(taskSet, tid, serializedData)
             } else if (Set(TaskState.FAILED, TaskState.KILLED, TaskState.LOST).contains(state)) {
               taskSet.removeRunningTask(tid)
               taskResultGetter.enqueueFailedTask(taskSet, tid, state, serializedData)