Skip to content

Commit

Permalink
[LIVY-495] Add thriftserver UI
Browse files Browse the repository at this point in the history
## What changes were proposed in this pull request?

The PR adds a new table in the Session tab dedicated to thrift-server sessions. The table contains the active sessions with a link to the corresponding Livy session.

Moreover, the same information is also exposed through a REST endpoint (`"thriftserver/sessions"`).

## How was this patch tested?

Manual tests. A screenshot of the UI is:

![screen shot 2018-10-09 at 11 32 18 am](https://user-images.githubusercontent.com/8821783/46660327-11082400-cbb7-11e8-9779-e8d85483dc97.png)

Author: Marco Gaido <[email protected]>

Closes #114 from mgaido91/LIVY-495.
  • Loading branch information
mgaido91 authored and Marcelo Vanzin committed Oct 25, 2018
1 parent 4fb5516 commit 0442eb0
Show file tree
Hide file tree
Showing 8 changed files with 222 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ function loadSessionsTable(sessions) {
tdWrap(session.kind) +
tdWrap(session.state) +
tdWrap(logLinks(session, "session")) +
"</tr>"
"</tr>"
);
});
}
Expand All @@ -39,7 +39,7 @@ function loadBatchesTable(sessions) {
tdWrap(appIdLink(session)) +
tdWrap(session.state) +
tdWrap(logLinks(session, "batch")) +
"</tr>"
"</tr>"
);
});
}
Expand Down
16 changes: 12 additions & 4 deletions server/src/main/scala/org/apache/livy/server/LivyServer.scala
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@ class LivyServer extends Logging {
livyConf.set(LIVY_SPARK_SCALA_VERSION.key,
sparkScalaVersion(formattedSparkVersion, scalaVersionFromSparkSubmit, livyConf))

val thriftServerFactory = if (livyConf.getBoolean(LivyConf.THRIFT_SERVER_ENABLED)) {
Some(ThriftServerFactory.getInstance)
} else {
None
}

if (UserGroupInformation.isSecurityEnabled) {
// If Hadoop security is enabled, run kinit periodically. runKinit() should be called
// before any Hadoop operation, otherwise Kerberos exception will be thrown.
Expand Down Expand Up @@ -211,10 +217,13 @@ class LivyServer extends Logging {
mount(context, batchServlet, "/batches/*")

if (livyConf.getBoolean(UI_ENABLED)) {
val uiServlet = new UIServlet(basePath)
val uiServlet = new UIServlet(basePath, livyConf)
mount(context, uiServlet, "/ui/*")
mount(context, staticResourceServlet, "/static/*")
mount(context, uiRedirectServlet(basePath + "/ui/"), "/*")
thriftServerFactory.foreach { factory =>
mount(context, factory.getServlet(basePath), factory.getServletMappings: _*)
}
} else {
mount(context, uiRedirectServlet(basePath + "/metrics"), "/*")
}
Expand Down Expand Up @@ -278,9 +287,8 @@ class LivyServer extends Logging {
}
})

if (livyConf.getBoolean(LivyConf.THRIFT_SERVER_ENABLED)) {
ThriftServerFactory.getInstance.start(
livyConf, interactiveSessionManager, sessionStore, accessManager)
thriftServerFactory.foreach {
_.start(livyConf, interactiveSessionManager, sessionStore, accessManager)
}

_serverUrl = Some(s"${server.protocol}://${server.host}:${server.port}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.livy.server

import javax.servlet.Servlet

import org.apache.livy.LivyConf
import org.apache.livy.server.recovery.SessionStore
import org.apache.livy.sessions.InteractiveSessionManager
Expand All @@ -30,6 +32,10 @@ trait ThriftServerFactory {
livySessionManager: InteractiveSessionManager,
sessionStore: SessionStore,
accessManager: AccessManager): Unit

def getServlet(basePath: String): Servlet

def getServletMappings: Seq[String]
}

object ThriftServerFactory {
Expand Down
64 changes: 39 additions & 25 deletions server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,40 @@ import scala.xml.Node

import org.scalatra.ScalatraServlet

class UIServlet(val basePath: String) extends ScalatraServlet {
import org.apache.livy.LivyConf

class UIServlet(val basePath: String, livyConf: LivyConf) extends ScalatraServlet {
before() { contentType = "text/html" }

sealed trait Page { val name: String }
private trait Page {
val name: String
def getNavCrumbs: Seq[Node] = Seq.empty
}
private case class SimplePage(name: String) extends Page
private case class AllSessionsPage(name: String = "Sessions") extends Page

private case class AllSessionsPage(name: String = "Sessions") extends Page {
override def getNavCrumbs: Seq[Node] = <li class="active"><a href="#">Sessions</a></li>
}
private case class SessionPage(id: Int) extends Page {
val name: String = "Session " + id
override def getNavCrumbs: Seq[Node] = {
<li><a href={basePath + "/ui"}>Sessions</a></li> ++
<li class="active"><a href="#">{name}</a></li>
}
}
private case class LogPage(sessionType: String, id: Int) extends Page {
val sessionName: String = sessionType + " " + id
val name: String = sessionName + " Log"
override def getNavCrumbs: Seq[Node] = {
val sessionLink = if (sessionType == "Session") {
basePath + "/ui/session/" + id
} else {
"#"
}
<li><a href={basePath + "/ui"}>Sessions</a></li> ++
<li><a href={sessionLink}>{sessionName}</a></li> ++
<li class="active"><a href="#">Log</a></li>
}
}

private def getHeader(pageName: String): Seq[Node] =
Expand All @@ -55,7 +77,7 @@ class UIServlet(val basePath: String) extends ScalatraServlet {
<title>Livy - {pageName}</title>
</head>

private def wrapNavTabs(tabs: Seq[Node]): Seq[Node] =
private def wrapNavCrumbs(crumbs: Seq[Node]): Seq[Node] =
<nav class="navbar navbar-default">
<div class="container-fluid">
<div class="navbar-header">
Expand All @@ -65,32 +87,12 @@ class UIServlet(val basePath: String) extends ScalatraServlet {
</div>
<div class="collapse navbar-collapse">
<ul class="nav navbar-nav">
{tabs}
{crumbs}
</ul>
</div>
</div>
</nav>

private def getNavBar(page: Page): Seq[Node] = {
val tabs: Seq[Node] = page match {
case _: AllSessionsPage => <li class="active"><a href="#">Sessions</a></li>
case sessionPage: SessionPage => {
<li><a href={basePath + "/ui"}>Sessions</a></li> ++
<li class="active"><a href="#">{sessionPage.name}</a></li>
}
case logPage: LogPage => {
val sessionLink = if (logPage.sessionType == "Session") {
basePath + "/ui/session/" + logPage.id
} else "#"
<li><a href={basePath + "/ui"}>Sessions</a></li> ++
<li><a href={sessionLink}>{logPage.sessionName}</a></li> ++
<li class="active"><a href="#">Log</a></li>
}
case _ => Seq.empty
}
wrapNavTabs(tabs)
}

private def createPage(pageInfo: Page, pageContents: Seq[Node]): Seq[Node] =
<html>
{getHeader(pageInfo.name)}
Expand All @@ -102,15 +104,27 @@ class UIServlet(val basePath: String) extends ScalatraServlet {
</body>
</html>

private def getNavBar(page: Page): Seq[Node] = wrapNavCrumbs(page.getNavCrumbs)

notFound {
createPage(SimplePage("404"), <h3>404 No Such Page</h3>)
}

private def thriftSessionsTable: Seq[Node] = {
if (livyConf.getBoolean(LivyConf.THRIFT_SERVER_ENABLED)) {
<div id="thrift-sessions"></div> ++
<script src={s"$basePath/static/js/thrift-sessions.js"}></script>
} else {
Seq.empty
}
}

get("/") {
val content =
<div id="all-sessions">
<div id="interactive-sessions"></div>
<div id="batches"></div>
{thriftSessionsTable}
<script src={basePath + "/static/js/all-sessions.js"}></script>
</div>

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<h4 class="sessions-template">JDBC/ODBC Open Sessions</h4>

<table id="thrift-sessions-table"
class="table table-striped sessions-table sessions-template">
<thead class="sessions-table-head">
<tr>
<th>
<span data-toggle="tooltip"
title="JDBC/ODBC session Id for the session">
Session Id
</span>
</th>
<th>
<span data-toggle="tooltip"
title="Livy Interactive Session Id for this session. Links to the Session Summary Page">
Livy Session Id
</span>
</th>
<th>
<span data-toggle="tooltip" title="Remote user who submitted this session">
Owner
</span>
</th>
<th>
<span data-toggle="tooltip"
title="Creation time of the session">
Created At
</span>
</th>
</tr>
</thead>
<tbody class="sessions-table-body">
</tbody>
</table>
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

function loadThriftSessionsTable(sessions) {
$.each(sessions, function(index, session) {
$("#thrift-sessions-table .sessions-table-body").append(
"<tr>" +
tdWrap(session.sessionId) +
tdWrap(uiLink("session/" + session.livySessionId, session.livySessionId)) +
tdWrap(session.owner) +
tdWrap(session.createdAt) +
"</tr>"
);
});
}

var numSessions = 0;

$(document).ready(function () {
var sessionsReq = $.getJSON(location.origin + prependBasePath("/thriftserver/sessions"), function(response) {
if (response && response.total > 0) {
$("#thrift-sessions").load(prependBasePath("/static/html/thrift-sessions-table.html .sessions-template"), function() {
loadThriftSessionsTable(response.sessions);
$("#thrift-sessions-table").DataTable();
$('#thrift-sessions [data-toggle="tooltip"]').tooltip();
});
}
numSessions = response.total;
});

$.when(sessionsReq).done(function () {
if (numSessions == 0) {
$("#thrift-sessions").append('<h4>No open JDBC/ODBC sessions.</h4>');
}
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@

package org.apache.livy.thriftserver

import javax.servlet.Servlet

import org.apache.livy.LivyConf
import org.apache.livy.server.{AccessManager, ThriftServerFactory}
import org.apache.livy.server.recovery.SessionStore
import org.apache.livy.sessions.InteractiveSessionManager
import org.apache.livy.thriftserver.ui.ThriftJsonServlet

class ThriftServerFactoryImpl extends ThriftServerFactory {
override def start(
Expand All @@ -34,4 +37,8 @@ class ThriftServerFactoryImpl extends ThriftServerFactory {
}
LivyThriftServer.start(livyConf, livySessionManager, sessionStore, accessManager)
}

override def getServlet(basePath: String): Servlet = new ThriftJsonServlet(basePath)

override def getServletMappings: Seq[String] = Seq("/thriftserver/*")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.livy.thriftserver.ui

import java.text.SimpleDateFormat

import org.apache.livy.server.JsonServlet
import org.apache.livy.thriftserver.LivyThriftServer


class ThriftJsonServlet(val basePath: String) extends JsonServlet {

private val df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z")

case class SessionInfo(
sessionId: String,
livySessionId: String,
owner: String,
createdAt: String)

get("/sessions") {
val thriftSessions = LivyThriftServer.getInstance.map { server =>
val sessionManager = server.getSessionManager()
sessionManager.getSessions.map { sessionHandle =>
val info = sessionManager.getSessionInfo(sessionHandle)
SessionInfo(sessionHandle.getSessionId.toString,
sessionManager.livySessionId(sessionHandle).map(_.toString).getOrElse(""),
info.username,
df.format(info.creationTime))
}.toSeq
}.getOrElse(Seq.empty)
val from = params.get("from").map(_.toInt).getOrElse(0)
val size = params.get("size").map(_.toInt).getOrElse(100)

Map(
"from" -> from,
"total" -> thriftSessions.length,
"sessions" -> thriftSessions.view(from, from + size))
}
}

0 comments on commit 0442eb0

Please sign in to comment.