selenium test spark, pyspark and sparkSql

### What is this PR for?
Test functionality of spark, pyspark, sparksql

### What type of PR is it?
Improvement

### Todos
* [x] - Selenium for spark
* [x] - Selenium for pyspark
* [x] - Selenium for sparksql
* [x] - refactor with apache#619

### Is there a relevant Jira issue?
ZEPPELIN-587

### How should this be tested?
On macOS

    PATH=~/Applications/Firefox.app/Contents/MacOS/:$PATH CI="" \
    mvn -Dtest=org.apache.zeppelin.integration.TestSparkParagraph -Denforcer.skip=true \
    test -pl zeppelin-server

Author: Prabhjyot Singh <[email protected]>

Closes apache#654 from prabhjyotsingh/ZEPPELIN-587 and squashes the following commits:

8f24695 [Prabhjyot Singh] use handleException in all other test cases remove test for spark 1.1.1 more meaningful log message
28dfc55 [Prabhjyot Singh] thorwong exception similar to apache#709
21bcc45 [Prabhjyot Singh] Merge remote-tracking branch 'origin/master' into ZEPPELIN-587
b05b81b [Prabhjyot Singh] have SHIFT_ENTER enum
9a206f4 [Prabhjyot Singh] add missing endToEndTestEnabled check for testSparkInterpreterDependencyLoading
ab03287 [Prabhjyot Singh] have static import of AbstractZeppelinIT.HelperKeys.*, and rg.openqa.selenium.Keys.*
5187a16 [Prabhjyot Singh] check if spark version is less than 1.3 then don't append ".toDF()"
6927f7e [Prabhjyot Singh] CI FIX
9236e3c [Prabhjyot Singh] implemeting @bzz review comments
2c28758 [Prabhjyot Singh] missed refactor with apache#619
d49344f [Prabhjyot Singh] selenium test spark, pyspark and sparkSql

zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java

@@ -19,8 +19,6 @@
 
 
 import com.google.common.base.Function;
-import org.junit.After;
-import org.junit.Before;
 import org.openqa.selenium.*;
 import org.openqa.selenium.support.ui.ExpectedConditions;
 import org.openqa.selenium.support.ui.FluentWait;
@@ -29,10 +27,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import static org.openqa.selenium.Keys.ENTER;
+import static org.openqa.selenium.Keys.SHIFT;
+
 abstract public class AbstractZeppelinIT {
   protected WebDriver driver;
 
@@ -131,4 +133,44 @@ protected void deleteTestNotebook(final WebDriver driver) {
     sleep(100, true);
   }
 
+  public enum HelperKeys implements CharSequence {
+    OPEN_PARENTHESIS(Keys.chord(Keys.SHIFT, "9")),
+    EXCLAMATION(Keys.chord(Keys.SHIFT, "1")),
+    PERCENTAGE(Keys.chord(Keys.SHIFT, "5")),
+    SHIFT_ENTER(Keys.chord(SHIFT, ENTER));
+
+    private final CharSequence keyCode;
+
+    HelperKeys(CharSequence keyCode) {
+      this.keyCode = keyCode;
+    }
+
+    public char charAt(int index) {
+      return index == 0 ? keyCode.charAt(index) : '\ue000';
+    }
+
+    public int length() {
+      return 1;
+    }
+
+    public CharSequence subSequence(int start, int end) {
+      if (start == 0 && end == 1) {
+        return String.valueOf(this.keyCode);
+      } else {
+        throw new IndexOutOfBoundsException();
+      }
+    }
+
+    public String toString() {
+      return String.valueOf(this.keyCode);
+    }
+
+  }
+
+  protected void handleException(String message, Exception e) throws Exception {
+    LOG.error(message, e);
+    File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
+    throw e;
+  }
+
 }

zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java

@@ -227,9 +227,7 @@ public void testMoveUpAndDown() throws Exception {
       deleteTestNotebook(driver);
 
     } catch (Exception e) {
-      LOG.error("Exception in ParagraphActionsIT while testMoveUpAndDown ", e);
-      File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
-      throw e;
+      handleException("Exception in ParagraphActionsIT while testMoveUpAndDown ", e);
     }
 
   }
@@ -267,9 +265,7 @@ public void testDisableParagraphRunButton() throws Exception {
       deleteTestNotebook(driver);
 
     } catch (Exception e) {
-      LOG.error("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e);
-      File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
-      throw e;
+      handleException("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e);
     }
 
   }

zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java

@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.integration;
+
+
+import org.apache.zeppelin.AbstractZeppelinIT;
+import org.apache.zeppelin.WebDriverManager;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ErrorCollector;
+import org.openqa.selenium.By;
+import org.openqa.selenium.TimeoutException;
+import org.openqa.selenium.WebElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.zeppelin.AbstractZeppelinIT.HelperKeys.*;
+import static org.openqa.selenium.Keys.*;
+
+public class SparkParagraphIT extends AbstractZeppelinIT {
+  private static final Logger LOG = LoggerFactory.getLogger(SparkParagraphIT.class);
+
+
+  @Rule
+  public ErrorCollector collector = new ErrorCollector();
+
+  @Before
+  public void startUp() {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    driver = WebDriverManager.getWebDriver();
+    createNewNote();
+    waitForParagraph(1, "READY");
+  }
+
+  @After
+  public void tearDown() {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    deleteTestNotebook(driver);
+    driver.quit();
+  }
+
+  @Test
+  public void testSpark() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+      paragraph1Editor.sendKeys("sc.version");
+      paragraph1Editor.sendKeys(SHIFT_ENTER);
+
+      waitForParagraph(1, "FINISHED");
+      WebElement paragraph1Result = driver.findElement(By.xpath(
+          getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]"));
+      Float sparkVersion = Float.parseFloat(paragraph1Result.getText().split("= ")[1].substring(0, 3));
+
+      WebElement paragraph2Editor = driver.findElement(By.xpath(getParagraphXPath(2) + "//textarea"));
+
+
+      /*
+      equivalent of
+      import org.apache.commons.io.IOUtils
+      import java.net.URL
+      import java.nio.charset.Charset
+      val bankText = sc.parallelize(IOUtils.toString(new URL("https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv"),Charset.forName("utf8")).split("\n"))
+      case class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)
+
+      val bank = bankText.map(s => s.split(";")).filter(s => s(0) != "\"age\"").map(s => Bank(s(0).toInt,s(1).replaceAll("\"", ""),s(2).replaceAll("\"", ""),s(3).replaceAll("\"", ""),s(5).replaceAll("\"", "").toInt)).toDF()
+      bank.registerTempTable("bank")
+       */
+      paragraph2Editor.sendKeys("import org.apache.commons.io.IOUtils" +
+          ENTER +
+
+          "import java.net.URL" +
+          ENTER +
+
+          "import java.nio.charset.Charset" +
+          ENTER +
+
+          "val bankText = sc.parallelize" + OPEN_PARENTHESIS +
+          "IOUtils.toString" + OPEN_PARENTHESIS + "new URL" + OPEN_PARENTHESIS
+          + "\"https://s3.amazonaws.com/apache" + SUBTRACT + "zeppelin/tutorial/bank/bank." +
+          "csv\"),Charset.forName" + OPEN_PARENTHESIS + "\"utf8\"))" +
+          ".split" + OPEN_PARENTHESIS + "\"\\n\"))" +
+          ENTER +
+
+          "case class Bank" + OPEN_PARENTHESIS +
+          "age: Integer, job: String, marital: String, education: String, balance: Integer)" +
+          ENTER +
+          ENTER +
+
+          "val bank = bankText.map" + OPEN_PARENTHESIS + "s => s.split" +
+          OPEN_PARENTHESIS + "\";\")).filter" + OPEN_PARENTHESIS +
+          "s => s" + OPEN_PARENTHESIS + "0) " + EXCLAMATION +
+          "= \"\\\"age\\\"\").map" + OPEN_PARENTHESIS +
+          "s => Bank" + OPEN_PARENTHESIS + "s" + OPEN_PARENTHESIS +
+          "0).toInt,s" + OPEN_PARENTHESIS + "1).replaceAll" +
+          OPEN_PARENTHESIS + "\"\\\"\", \"\")," +
+          "s" + OPEN_PARENTHESIS + "2).replaceAll" +
+          OPEN_PARENTHESIS + "\"\\\"\", \"\")," +
+          "s" + OPEN_PARENTHESIS + "3).replaceAll" +
+          OPEN_PARENTHESIS + "\"\\\"\", \"\")," +
+          "s" + OPEN_PARENTHESIS + "5).replaceAll" +
+          OPEN_PARENTHESIS + "\"\\\"\", \"\").toInt" + ")" +
+          ")" + (sparkVersion < 1.3f ? "" : ".toDF" + OPEN_PARENTHESIS + ")") +
+          ENTER +
+
+          "bank.registerTempTable" + OPEN_PARENTHESIS + "\"bank\")"
+      );
+      paragraph2Editor.sendKeys("" + END + BACK_SPACE + BACK_SPACE +
+          BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE +
+          BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE);
+
+      paragraph2Editor.sendKeys(SHIFT_ENTER);
+
+      try {
+        waitForParagraph(2, "FINISHED");
+      } catch (TimeoutException e) {
+        waitForParagraph(2, "ERROR");
+        collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark status:",
+            "ERROR", CoreMatchers.equalTo("FINISHED")
+        );
+      }
+
+      WebElement paragraph2Result = driver.findElement(By.xpath(
+          getParagraphXPath(2) + "//div[@class=\"tableDisplay\"]"));
+
+      collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark result: ",
+          paragraph2Result.getText().toString(), CoreMatchers.containsString(
+              "import org.apache.commons.io.IOUtils"
+          )
+      );
+
+    } catch (Exception e) {
+      handleException("Exception in SparkParagraphIT while testSpark", e);
+    }
+  }
+
+  @Test
+  public void testPySpark() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+
+      paragraph1Editor.sendKeys(PERCENTAGE + "pyspark" + ENTER +
+          "for x in range" + OPEN_PARENTHESIS + "0, 3):" + ENTER +
+          "    print \"test loop " + PERCENTAGE + "d\" " +
+          PERCENTAGE + " " + OPEN_PARENTHESIS + "x)" + ENTER);
+
+      paragraph1Editor.sendKeys(SHIFT_ENTER);
+
+      try {
+        waitForParagraph(1, "FINISHED");
+      } catch (TimeoutException e) {
+        waitForParagraph(1, "ERROR");
+        collector.checkThat("Paragraph from SparkParagraphIT of testPySpark status: ",
+            "ERROR", CoreMatchers.equalTo("FINISHED")
+        );
+      }
+
+      WebElement paragraph1Result = driver.findElement(By.xpath(
+          getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]"));
+      collector.checkThat("Paragraph from SparkParagraphIT of testPySpark result: ",
+          paragraph1Result.getText().toString(), CoreMatchers.equalTo("test loop 0\ntest loop 1\ntest loop 2")
+      );
+
+    } catch (Exception e) {
+      handleException("Exception in SparkParagraphIT while testPySpark", e);
+    }
+  }
+
+  @Test
+  public void testSqlSpark() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+
+      paragraph1Editor.sendKeys(PERCENTAGE + "sql" + ENTER +
+          "select * from bank limit 1");
+
+      paragraph1Editor.sendKeys(SHIFT_ENTER);
+
+      try {
+        waitForParagraph(1, "FINISHED");
+      } catch (TimeoutException e) {
+        waitForParagraph(1, "ERROR");
+        collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark status: ",
+            "ERROR", CoreMatchers.equalTo("FINISHED")
+        );
+      }
+
+      WebElement paragraph1Result = driver.findElement(By.xpath(
+          getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]"));
+      collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
+          paragraph1Result.getText().toString(), CoreMatchers.equalTo("age job marital education balance\n" +
+              "30 unemployed married primary 1,787")
+      );
+    } catch (Exception e) {
+      handleException("Exception in SparkParagraphIT while testSqlSpark", e);
+    }
+  }
+}

zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java

@@ -196,9 +196,7 @@ public void testAngularDisplay() throws Exception {
 
       System.out.println("testCreateNotebook Test executed");
     } catch (Exception e) {
-      LOG.error("Exception in ZeppelinIT while testAngularDisplay ", e);
-      File scrFile = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
-      throw e;
+      handleException("Exception in ZeppelinIT while testAngularDisplay ", e);
     }
   }
 
@@ -254,9 +252,7 @@ public void testSparkInterpreterDependencyLoading() throws Exception {
       driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
       driver.switchTo().alert().accept();
     } catch (Exception e) {
-      LOG.error("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
-      File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
-      throw e;
+      handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
     }
   }
 }