/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.integration;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.openqa.selenium.By;
import org.openqa.selenium.TimeoutException;
import org.openqa.selenium.WebElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SparkParagraphIT extends AbstractZeppelinIT {
private static final Logger LOG = LoggerFactory.getLogger(SparkParagraphIT.class);
@Rule
public ErrorCollector collector = new ErrorCollector();
@Before
public void startUp() {
if (!endToEndTestEnabled()) {
return;
}
driver = WebDriverManager.getWebDriver();
createNewNote();
waitForParagraph(1, "READY");
}
@After
public void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
deleteTestNotebook(driver);
driver.quit();
}
@Test
public void testSpark() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
setTextOfParagraph(1, "sc.version");
runParagraph(1);
waitForParagraph(1, "FINISHED");
/*
equivalent of
import org.apache.commons.io.IOUtils
import java.net.URL
import java.nio.charset.Charset
val bankText = sc.parallelize(IOUtils.toString(new URL("https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv"),Charset.forName("utf8")).split("\n"))
case class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)
val bank = bankText.map(s => s.split(";")).filter(s => s(0) != "\"age\"").map(s => Bank(s(0).toInt,s(1).replaceAll("\"", ""),s(2).replaceAll("\"", ""),s(3).replaceAll("\"", ""),s(5).replaceAll("\"", "").toInt)).toDF()
bank.registerTempTable("bank")
*/
setTextOfParagraph(2, "import org.apache.commons.io.IOUtils\\n" +
"import java.net.URL\\n" +
"import java.nio.charset.Charset\\n" +
"val bankText = sc.parallelize(IOUtils.toString(new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),Charset.forName(\"utf8\")).split(\"\\\\n\"))\\n" +
"case class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\\n" +
"\\n" +
"val bank = bankText.map(s => s.split(\";\")).filter(s => s(0) != \"\\\\\"age\\\\\"\").map(s => Bank(s(0).toInt,s(1).replaceAll(\"\\\\\"\", \"\"),s(2).replaceAll(\"\\\\\"\", \"\"),s(3).replaceAll(\"\\\\\"\", \"\"),s(5).replaceAll(\"\\\\\"\", \"\").toInt)).toDF()\\n" +
"bank.registerTempTable(\"bank\")");
runParagraph(2);
try {
waitForParagraph(2, "FINISHED");
} catch (TimeoutException e) {
waitForParagraph(2, "ERROR");
collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark status:",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
WebElement paragraph2Result = driver.findElement(By.xpath(
getParagraphXPath(2) + "//div[contains(@id,\"_text\")]"));
collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark result: ",
paragraph2Result.getText().toString(), CoreMatchers.containsString(
"import org.apache.commons.io.IOUtils"
)
);
} catch (Exception e) {
handleException("Exception in SparkParagraphIT while testSpark", e);
}
}
@Test
public void testPySpark() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
setTextOfParagraph(1, "%pyspark\\n" +
"for x in range(0, 3):\\n" +
" print \"test loop %d\" % (x)");
runParagraph(1);
try {
waitForParagraph(1, "FINISHED");
} catch (TimeoutException e) {
waitForParagraph(1, "ERROR");
collector.checkThat("Paragraph from SparkParagraphIT of testPySpark status: ",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
WebElement paragraph1Result = driver.findElement(By.xpath(
getParagraphXPath(1) + "//div[contains(@id,\"_text\")]"));
collector.checkThat("Paragraph from SparkParagraphIT of testPySpark result: ",
paragraph1Result.getText().toString(), CoreMatchers.equalTo("test loop 0\ntest loop 1\ntest loop 2")
);
// the last statement's evaluation result is printed
setTextOfParagraph(2, "%pyspark\\n" +
"sc.version\\n" +
"1+1");
runParagraph(2);
try {
waitForParagraph(2, "FINISHED");
} catch (TimeoutException e) {
waitForParagraph(2, "ERROR");
collector.checkThat("Paragraph from SparkParagraphIT of testPySpark status: ",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
WebElement paragraph2Result = driver.findElement(By.xpath(
getParagraphXPath(2) + "//div[contains(@id,\"_text\")]"));
collector.checkThat("Paragraph from SparkParagraphIT of testPySpark result: ",
paragraph2Result.getText().toString(), CoreMatchers.equalTo("2")
);
} catch (Exception e) {
handleException("Exception in SparkParagraphIT while testPySpark", e);
}
}
@Test
public void testSqlSpark() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
setTextOfParagraph(1,"%sql\\n" +
"select * from bank limit 1");
runParagraph(1);
try {
waitForParagraph(1, "FINISHED");
} catch (TimeoutException e) {
waitForParagraph(1, "ERROR");
collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark status: ",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
WebElement paragraph1Result = driver.findElement(By.xpath(
getParagraphXPath(1) + "//div[contains(@id,\"_graph\")]/div/div/div/div/div[1]"));
collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
paragraph1Result.getText().toString(), CoreMatchers.equalTo("age\n▼\njob\n▼\nmarital\n▼\neducation\n▼\nbalance\n▼\n30 unemployed married primary 1787"));
} catch (Exception e) {
handleException("Exception in SparkParagraphIT while testSqlSpark", e);
}
}
@Test
public void testDep() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// restart spark interpreter before running %dep
clickAndWait(By.xpath("//span[@uib-tooltip='Interpreter binding']"));
clickAndWait(By.xpath("//div[font[contains(text(), 'spark')]]/preceding-sibling::a[@uib-tooltip='Restart']"));
clickAndWait(By.xpath("//button[contains(.,'OK')]"));
setTextOfParagraph(1,"%dep z.load(\"org.apache.commons:commons-csv:1.1\")");
runParagraph(1);
try {
waitForParagraph(1, "FINISHED");
WebElement paragraph1Result = driver.findElement(By.xpath(getParagraphXPath(1) +
"//div[contains(@id,'_text')]"));
collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
paragraph1Result.getText(), CoreMatchers.containsString("res0: org.apache.zeppelin.dep.Dependency = org.apache.zeppelin.dep.Dependency"));
setTextOfParagraph(2, "import org.apache.commons.csv.CSVFormat");
runParagraph(2);
try {
waitForParagraph(2, "FINISHED");
WebElement paragraph2Result = driver.findElement(By.xpath(getParagraphXPath(2) +
"//div[contains(@id,'_text')]"));
collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
paragraph2Result.getText(), CoreMatchers.equalTo("import org.apache.commons.csv.CSVFormat"));
} catch (TimeoutException e) {
waitForParagraph(2, "ERROR");
collector.checkThat("Second paragraph from SparkParagraphIT of testDep status: ",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
} catch (TimeoutException e) {
waitForParagraph(1, "ERROR");
collector.checkThat("First paragraph from SparkParagraphIT of testDep status: ",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
} catch (Exception e) {
handleException("Exception in SparkParagraphIT while testDep", e);
}
}
}