[SPARK-3468] [WEBUI] Timeline-View feature

I sometimes trouble-shoot and analyse the cause of long time spending job. At the time, I find the stages which spends long time or fails, then I find the tasks which spends long time or fails, next I analyse the proportion of each phase in a task. Another case, I find executors which spends long time for running a task and analyse the details of a task. In such situation, I think it's helpful to visualize timeline for each application, job, task and the details of proportion of activity for each task. I added 3 timeline features to existing Web UI. [Application Timeline View] This view shows following things. * When each executor was added/removed and the reason why it's removed. * When each job was started/finished. * Status of each job. ![screenshot from 2015-04-01 16 49 25](https://cloud.githubusercontent.com/assets/4736016/6936886/e35fd582-d891-11e4-980d-8de13f50e442.png) [Stage Timeline View] Similar to Application Timeline View, this view shows following things. * When each executor was added/removed and the reason why it's removed. * When each job was started/finished. * Status of each stage. ![screenshot from 2015-04-01 16 50 59](https://cloud.githubusercontent.com/assets/4736016/6936900/0dca6526-d892-11e4-84a8-efd9037af444.png) [Task Assignment Timeline View] This view shows following things. * When each task started/finished * How long each task spent and the proportion. * Status of each task. * Where each task ran on. ![screenshot from 2015-04-01 16 51 54](https://cloud.githubusercontent.com/assets/4736016/6936910/20fd5acc-d892-11e4-9018-80e463881fc2.png) All the view above is zoomable by mouse wheel action and scrollable by drag action. Author: Kousuke Saruta <[email protected]> Closes apache#2342 from sarutak/timeline-viewer-feature and squashes the following commits: 11fe67d [Kousuke Saruta] Fixed conflict 79ac03d [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature a91abd3 [Kousuke Saruta] Merge branch 'master' of https://github.com/apache/spark into timeline-viewer-feature ef34a5b [Kousuke Saruta] Implement tooltip using bootstrap b09d0c5 [Kousuke Saruta] Move `stroke` and `fill` attribute of rect elements to css d3c63c8 [Kousuke Saruta] Fixed a little bit bugs a36291b [Kousuke Saruta] Merge branch 'master' of https://github.com/apache/spark into timeline-viewer-feature 28714b6 [Kousuke Saruta] Fixed highlight issue 0dc4278 [Kousuke Saruta] Addressed most of Patrics's feedbacks 8110acf [Kousuke Saruta] Added scroll limit to Job timeline 974a64a [Kousuke Saruta] Removed unused function ee7a7f0 [Kousuke Saruta] Refactored 6a91872 [Kousuke Saruta] Temporary commit 6693f34 [Kousuke Saruta] Added link to job/stage box in the timeline in order to move to corresponding row when we click 8f88222 [Kousuke Saruta] Added job/stage description aeed4b1 [Kousuke Saruta] Removed stage timeline fc1696c [Kousuke Saruta] Merge branch 'timeline-viewer-feature' of github.com:sarutak/spark into timeline-viewer-feature 999ccd4 [Kousuke Saruta] Improved scalability 0fc6a31 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature 19815ae [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature 68b7540 [Kousuke Saruta] Merge branch 'timeline-viewer-feature' of github.com:sarutak/spark into timeline-viewer-feature 52b5f0b [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature dec85db [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature fcdab7d [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature dab7cc1 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature 09cce97 [Kousuke Saruta] Cleanuped 16f82cf [Kousuke Saruta] Cleanuped 9fb522e [Kousuke Saruta] Cleanuped d05f2c2 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature e85e9aa [Kousuke Saruta] Cleanup: Added TimelineViewUtils.scala a76e569 [Kousuke Saruta] Removed unused setting in timeline-view.css 5ce1b21 [Kousuke Saruta] Added vis.min.js, vis.min.css and vis.map to .rat-exclude 082f709 [Kousuke Saruta] Added Timeline-View feature for Applications, Jobs and Stages
szhang-aa · May 1, 2015 · 7fe0f3f · 7fe0f3f
1 parent c24aeb6
commit 7fe0f3f
Show file tree

Hide file tree

Showing 15 changed files with 785 additions and 32 deletions.
diff --git a/.rat-excludes b/.rat-excludes
@@ -30,6 +30,9 @@ log4j-defaults.properties
 bootstrap-tooltip.js
 jquery-1.11.1.min.js
 sorttable.js
+vis.min.js
+vis.min.css
+vis.map
 .*avsc
 .*txt
 .*json

diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+div#application-timeline, div#job-timeline {
+  margin-bottom: 30px;
+}
+
+#application-timeline div.legend-area {
+  margin-top: 5px;
+}
+
+.vis.timeline div.content {
+  width: 100%;
+}
+
+.vis.timeline .item.stage {
+  cursor: pointer;
+}
+
+.vis.timeline .item.stage.succeeded {
+  background-color: #D5DDF6;
+}
+
+.vis.timeline .item.stage.succeeded.selected {
+   background-color: #D5DDF6;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.completed-stage-legend {
+  fill: #D5DDF6;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.stage.failed {
+   background-color: #FF5475;
+}
+
+.vis.timeline .item.stage.failed.selected {
+   background-color: #FF5475;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.failed-stage-legend {
+  fill: #FF5475;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.stage.running {
+   background-color: #FDFFCA;
+}
+
+.vis.timeline .item.stage.running.selected {
+   background-color: #FDFFCA;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.active-stage-legend {
+  fill: #FDFFCA;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.job {
+  cursor: pointer;
+}
+
+.vis.timeline .item.job.succeeded {
+  background-color: #D5DDF6;
+}
+
+.vis.timeline .item.job.succeeded.selected {
+   background-color: #D5DDF6;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.succeeded-job-legend {
+  fill: #D5DDF6;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.job.failed {
+   background-color: #FF5475;
+}
+
+.vis.timeline .item.job.failed.selected {
+   background-color: #FF5475;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.failed-job-legend {
+  fill: #FF5475;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.job.running {
+   background-color: #FDFFCA;
+}
+
+.vis.timeline .item.job.running.selected {
+   background-color: #FDFFCA;
+   border-color: #97B0F8;
+   z-index: auto;
+}
+
+.legend-area rect.running-job-legend {
+  fill: #FDFFCA;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.added {
+  background-color: #D5DDF6;
+}
+
+.legend-area rect.executor-added-legend {
+  fill: #D5DDF6;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.removed {
+  background-color: #EBCA59;
+}
+
+.legend-area rect.executor-removed-legend {
+  fill: #EBCA59;
+  stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.selected {
+  border-color: #FFC200;
+  background-color: #FFF785;
+  z-index: 2;
+}
+
+tr.corresponding-item-hover>td, tr.corresponding-item-hover>th {
+  background-color: #FFE1FA !important;
+}
+
+#application-timeline.collapsed {
+  display: none;
+}
+
+#job-timeline.collapsed {
+  display: none;
+}
+
+.control-panel {
+  margin-bottom: 5px;
+}
+
+span.expand-application-timeline, span.expand-job-timeline {
+  cursor: pointer;
+}
+
+.control-panel input+span {
+  cursor: pointer;
+}
+
+.vis.timeline .item.range .content {
+  position: unset;
+}
+
+.vis.timeline .item .tooltip-inner {
+  max-width: unset !important;
+}
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function drawApplicationTimeline(groupArray, eventObjArray, startTime) {
+  var groups = new vis.DataSet(groupArray);
+  var items = new vis.DataSet(eventObjArray);
+  var container = $("#application-timeline")[0];
+  var options = {
+    groupOrder: function(a, b) {
+      return a.value - b.value
+    },
+    editable: false,
+    showCurrentTime: false,
+    min: startTime,
+    zoomable: false
+  };
+
+  var applicationTimeline = new vis.Timeline(container);
+  applicationTimeline.setOptions(options);
+  applicationTimeline.setGroups(groups);
+  applicationTimeline.setItems(items);
+
+  setupZoomable("#application-timeline-zoom-lock", applicationTimeline);
+  setupExecutorEventAction();
+
+  function setupJobEventAction() {
+    $(".item.range.job.application-timeline-object").each(function() {
+      var getJobId = function(baseElem) {
+        var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
+        var jobId = jobIdText.match("\\(Job (\\d+)\\)")[1];
+       return jobId;
+      };
+
+      $(this).click(function() {
+        window.location.href = "job/?id=" + getJobId(this);
+      });
+
+      $(this).hover(
+        function() {
+          $("#job-" + getJobId(this)).addClass("corresponding-item-hover");
+          $($(this).find("div.application-timeline-content")[0]).tooltip("show");
+        },
+        function() {
+          $("#job-" + getJobId(this)).removeClass("corresponding-item-hover");
+          $($(this).find("div.application-timeline-content")[0]).tooltip("hide");
+        }
+      );
+    });
+  }
+
+  setupJobEventAction();
+
+  $("span.expand-application-timeline").click(function() {
+    $("#application-timeline").toggleClass('collapsed');
+
+    // Switch the class of the arrow from open to closed.
+    $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-open');
+    $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-closed');
+  });
+}
+
+function drawJobTimeline(groupArray, eventObjArray, startTime) {
+  var groups = new vis.DataSet(groupArray);
+  var items = new vis.DataSet(eventObjArray);
+  var container = $('#job-timeline')[0];
+  var options = {
+    groupOrder: function(a, b) {
+      return a.value - b.value;
+    },
+    editable: false,
+    showCurrentTime: false,
+    min: startTime,
+    zoomable: false,
+  };
+
+  var jobTimeline = new vis.Timeline(container);
+  jobTimeline.setOptions(options);
+  jobTimeline.setGroups(groups);
+  jobTimeline.setItems(items);
+
+  setupZoomable("#job-timeline-zoom-lock", jobTimeline);
+  setupExecutorEventAction();
+
+  function setupStageEventAction() {
+    $(".item.range.stage.job-timeline-object").each(function() {
+      var getStageIdAndAttempt = function(baseElem) {
+        var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
+        var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)")[1].split(".");
+        return stageIdAndAttempt;
+      };
+
+      $(this).click(function() {
+        var idAndAttempt = getStageIdAndAttempt(this);
+        var id = idAndAttempt[0];
+        var attempt = idAndAttempt[1];
+        window.location.href = "../../stages/stage/?id=" + id + "&attempt=" + attempt;
+      });
+
+      $(this).hover(
+        function() {
+          var idAndAttempt = getStageIdAndAttempt(this);
+          var id = idAndAttempt[0];
+          var attempt = idAndAttempt[1];
+          $("#stage-" + id + "-" + attempt).addClass("corresponding-item-hover");
+          $($(this).find("div.job-timeline-content")[0]).tooltip("show");
+        },
+        function() {
+          var idAndAttempt = getStageIdAndAttempt(this);
+          var id = idAndAttempt[0];
+          var attempt = idAndAttempt[1];
+          $("#stage-" + id + "-" + attempt).removeClass("corresponding-item-hover");
+          $($(this).find("div.job-timeline-content")[0]).tooltip("hide");
+        }
+      );
+    });
+  }
+
+  setupStageEventAction();
+
+  $("span.expand-job-timeline").click(function() {
+    $("#job-timeline").toggleClass('collapsed');
+
+    // Switch the class of the arrow from open to closed.
+    $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-open');
+    $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-closed');
+  });
+}
+
+function setupExecutorEventAction() {
+  $(".item.box.executor").each(function () {
+    $(this).hover(
+      function() {
+        $($(this).find(".executor-event-content")[0]).tooltip("show");
+      },
+      function() {
+        $($(this).find(".executor-event-content")[0]).tooltip("hide");
+      }
+    );
+  });
+}
+
+function setupZoomable(id, timeline) {
+  $(id + '>input[type="checkbox"]').click(function() {
+    if (this.checked) {
+      timeline.setOptions({zoomable: false});
+    } else {
+      timeline.setOptions({zoomable: true});
+    }
+  });
+
+  $(id + ">span").click(function() {
+    $(this).parent().find('input:checkbox').trigger('click');
+  });
+}
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis.map b/core/src/main/resources/org/apache/spark/ui/static/vis.map