LIHADOOP-20392: NullPointerException on zero mappers or reducers

rramos · May 27, 2016 · 12ea974 · 12ea974
1 parent d8baeb5
commit 12ea974
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 1 deletion.
diff --git a/app/com/linkedin/drelephant/mapreduce/MapReduceMetricsAggregator.java b/app/com/linkedin/drelephant/mapreduce/MapReduceMetricsAggregator.java
@@ -57,6 +57,12 @@ public void aggregate(HadoopApplicationData hadoopData) {
 
  long reduceIdealStartTime = mapTasks.getNthPercentileFinishTime(reduceTaskSlowStartPercentage);
 
+ // Mappers list is empty
+ if(reduceIdealStartTime == -1) {
+ // ideal start time for reducer is infinite since it cannot start
+ reduceIdealStartTime = Long.MAX_VALUE;
+ }
+
  reduceTasks = new TaskLevelAggregatedMetrics(data.getReducerData(), reduceTaskContainerSize, reduceIdealStartTime);
 
  _hadoopAggregatedData.setResourceUsed(mapTasks.getResourceUsed() + reduceTasks.getResourceUsed());

diff --git a/app/com/linkedin/drelephant/mapreduce/TaskLevelAggregatedMetrics.java b/app/com/linkedin/drelephant/mapreduce/TaskLevelAggregatedMetrics.java
@@ -46,6 +46,9 @@ public class TaskLevelAggregatedMetrics {
  */
  public long getNthPercentileFinishTime(int percentile)
  {
+ if(finishTimes == null || finishTimes.size() == 0 ) {
+ return -1;
+ }
  return Statistics.percentile(finishTimes, percentile);
  }
 
@@ -98,7 +101,7 @@ private void compute(MapReduceTaskData[] taskDatas, long containerSize, long ide
  long taskDurationMax = 0;
 
  // if there are zero tasks, then nothing to compute.
- if(taskDatas.length == 0) {
+ if(taskDatas == null || taskDatas.length == 0) {
  return;
  }
 
@@ -133,6 +136,11 @@ private void compute(MapReduceTaskData[] taskDatas, long containerSize, long ide
  // Compute the delay in starting the task.
  _delay = taskFinishTimeMax - (idealStartTime + taskDurationMax);
 
+ // invalid delay
+ if(_delay < 0) {
+ _delay = 0;
+ }
+
  // wastedResources
  long wastedMemory = containerSize - (long) (peakMemoryNeed * MEMORY_BUFFER); // give a 50% buffer
  if(wastedMemory > 0) {

diff --git a/public/images/runtime.png b/public/images/runtime.png
diff --git a/public/images/usedmemory.png b/public/images/usedmemory.png
diff --git a/public/images/waittime.png b/public/images/waittime.png
diff --git a/public/images/wastedmemory.png b/public/images/wastedmemory.png
diff --git a/test/com/linkedin/drelephant/mapreduce/TestTaskLevelAggregatedMetrics.java b/test/com/linkedin/drelephant/mapreduce/TestTaskLevelAggregatedMetrics.java
@@ -32,6 +32,14 @@ public void testZeroTasks() {
  Assert.assertEquals(taskMetrics.getResourceWasted(), 0);
  }
 
+ @Test
+ public void testNullTaskArray() {
+ TaskLevelAggregatedMetrics taskMetrics = new TaskLevelAggregatedMetrics(null, 0, 0);
+ Assert.assertEquals(taskMetrics.getDelay(), 0);
+ Assert.assertEquals(taskMetrics.getResourceUsed(), 0);
+ Assert.assertEquals(taskMetrics.getResourceWasted(), 0);
+ }
+
  @Test
  public void testTaskLevelData() {
  MapReduceTaskData taskData[] = new MapReduceTaskData[2];