From 54fdf0d8073a9d96c3b7cfb90af3f88c53fd3939 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 10:56:43 -0700 Subject: [PATCH 01/32] Separate spark-dependency into submodule --- pom.xml | 1 + spark-dependencies/pom.xml | 779 +++++++++++++++++++++++++++++++++++++ spark/pom.xml | 655 +------------------------------ 3 files changed, 787 insertions(+), 648 deletions(-) create mode 100644 spark-dependencies/pom.xml diff --git a/pom.xml b/pom.xml index 9b7cd4610be..ff651603f0e 100755 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,7 @@ zeppelin-interpreter zeppelin-zengine + spark-dependencies spark markdown angular diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml new file mode 100644 index 00000000000..1e48d6b4d85 --- /dev/null +++ b/spark-dependencies/pom.xml @@ -0,0 +1,779 @@ + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.6.0-incubating-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-spark-dependencies + jar + 0.6.0-incubating-SNAPSHOT + Zeppelin: Spark dependencies + Zeppelin spark support + http://zeppelin.incubator.apache.org + + + + 1.4.1 + 2.10.4 + 2.10 + + 2.3.0 + ${hadoop.version} + 1.7.7 + + 0.7.1 + 2.4.1 + + org.spark-project.akka + 2.3.4-spark + + http://www.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz + + + + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + + + + + org.apache.avro + avro + ${avro.version} + + + org.apache.avro + avro-ipc + ${avro.version} + + + io.netty + netty + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + servlet-api + + + org.apache.velocity + velocity + + + + + org.apache.avro + avro-mapred + ${avro.version} + ${avro.mapred.classifier} + + + io.netty + netty + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + servlet-api + + + org.apache.velocity + velocity + + + + + + + net.java.dev.jets3t + jets3t + ${jets3t.version} + runtime + + + commons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + commons-logging + commons-logging + + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + commons-logging + commons-logging + + + + + + org.apache.hadoop + hadoop-yarn-server-web-proxy + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + commons-logging + commons-logging + + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + commons-logging + commons-logging + + + + + + + + + + org.apache.spark + spark-core_2.10 + ${spark.version} + + + org.apache.hadoop + hadoop-client + + + + + + org.apache.spark + spark-repl_2.10 + ${spark.version} + + + + org.apache.spark + spark-sql_2.10 + ${spark.version} + + + + org.apache.spark + spark-hive_2.10 + ${spark.version} + + + + + org.apache.spark + spark-streaming_2.10 + ${spark.version} + + + + + org.apache.spark + spark-streaming-twitter_2.10 + ${spark.version} + + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + + ${akka.group} + akka-actor_${scala.binary.version} + ${akka.version} + + + ${akka.group} + akka-remote_${scala.binary.version} + ${akka.version} + + + ${akka.group} + akka-slf4j_${scala.binary.version} + ${akka.version} + + + ${akka.group} + akka-testkit_${scala.binary.version} + ${akka.version} + + + ${akka.group} + akka-zeromq_${scala.binary.version} + ${akka.version} + + + ${akka.group} + akka-actor_${scala.binary.version} + + + + + + + + + spark-1.1 + + + + + 1.1.1 + 2.2.3-shaded-protobuf + + + + + cassandra-spark-1.1 + + + com.datastax.spark + spark-cassandra-connector_${scala.binary.version} + 1.1.1 + + + org.joda + joda-convert + + + + + + 1.1.1 + 2.2.3-shaded-protobuf + + + + + spark-1.2 + + + + 1.2.1 + + + + + cassandra-spark-1.2 + + 1.2.1 + + + + com.datastax.spark + spark-cassandra-connector_${scala.binary.version} + 1.2.1 + + + org.joda + joda-convert + + + + + + + + spark-1.3 + + + 1.3.1 + + + + + + + + + cassandra-spark-1.3 + + 1.3.0 + + + + + com.datastax.spark + spark-cassandra-connector_${scala.binary.version} + + 1.3.0-SNAPSHOT + + + org.joda + joda-convert + + + + + + + + spark-1.4 + + 1.4.1 + + + + + + + + hadoop-0.23 + + + + org.apache.avro + avro + + + + 0.23.10 + + + + + hadoop-1 + + 1.0.4 + hadoop1 + 1.8.8 + org.spark-project.akka + + + + + hadoop-2.2 + + 2.2.0 + 2.5.0 + hadoop2 + + + + + hadoop-2.3 + + 2.3.0 + 2.5.0 + 0.9.3 + hadoop2 + + + + + hadoop-2.4 + + 2.4.0 + 2.5.0 + 0.9.3 + hadoop2 + + + + + hadoop-2.6 + + 2.6.0 + 2.5.0 + 0.9.3 + hadoop2 + + + + + mapr3 + + false + + + 1.0.3-mapr-3.0.3 + 2.3.0-mapr-4.0.0-FCS + 0.7.1 + + + + + mapr4 + + false + + + 2.3.0-mapr-4.0.0-FCS + 2.3.0-mapr-4.0.0-FCS + 0.7.1 + + + + org.apache.curator + curator-recipes + 2.4.0 + + + org.apache.zookeeper + zookeeper + + + + + org.apache.zookeeper + zookeeper + 3.4.5-mapr-1406 + + + + + + yarn + + + org.apache.spark + spark-yarn_2.10 + ${spark.version} + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + + + + pyspark + + http://www.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz + + + + + + com.googlecode.maven-download-plugin + download-maven-plugin + 1.2.1 + + + download-pyspark-files + validate + + wget + + + ${spark.download.url} + true + ${project.build.directory}/spark-dist + + + + + + maven-clean-plugin + + + + ${basedir}/../python/build + + + ${project.build.directory}/spark-dist + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + download-and-zip-pyspark-files + generate-resources + + run + + + + + + + + + + + + + + + + + + + + org.apache.rat + apache-rat-plugin + + + **/.idea/ + **/*.iml + .gitignore + **/.settings/* + **/.classpath + **/.project + **/target/** + **/derby.log + **/metastore_db/ + **/README.md + dependency-reduced-pom.xml + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.17 + + 1 + false + -Xmx1024m -XX:MaxPermSize=256m + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + *:* + + org/datanucleus/** + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + reference.conf + + + + + + package + + shade + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/spark/dep + false + false + true + org.datanucleus + + + + package + + copy + + + ${project.build.directory}/../../interpreter/spark/dep + false + false + true + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + diff --git a/spark/pom.xml b/spark/pom.xml index aa076877374..1a43db5052e 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -36,21 +36,7 @@ - 1.4.1 - 2.10.4 2.10 - - 2.3.0 - ${hadoop.version} - 1.7.7 - - 0.7.1 - 2.4.1 - - org.spark-project.akka - 2.3.4-spark - - http://www.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz @@ -59,192 +45,6 @@ https://repository.cloudera.com/artifactory/cloudera-repos/ - - - - - org.apache.avro - avro - ${avro.version} - - - org.apache.avro - avro-ipc - ${avro.version} - - - io.netty - netty - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - org.mortbay.jetty - servlet-api - - - org.apache.velocity - velocity - - - - - org.apache.avro - avro-mapred - ${avro.version} - ${avro.mapred.classifier} - - - io.netty - netty - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - org.mortbay.jetty - servlet-api - - - org.apache.velocity - velocity - - - - - - - net.java.dev.jets3t - jets3t - ${jets3t.version} - runtime - - - commons-logging - commons-logging - - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - commons-logging - commons-logging - - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - commons-logging - commons-logging - - - - - - org.apache.hadoop - hadoop-yarn-server-web-proxy - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - commons-logging - commons-logging - - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - commons-logging - commons-logging - - - - - @@ -264,104 +64,13 @@ provided - - - org.apache.spark - spark-core_2.10 - ${spark.version} - - - org.apache.hadoop - hadoop-client - - - - - - org.apache.spark - spark-repl_2.10 - ${spark.version} - - - - org.apache.spark - spark-sql_2.10 - ${spark.version} - - - - org.apache.spark - spark-hive_2.10 - ${spark.version} - - - - - org.apache.spark - spark-streaming_2.10 - ${spark.version} - - - - - org.apache.spark - spark-streaming-twitter_2.10 - ${spark.version} - - - - org.apache.spark - spark-catalyst_${scala.binary.version} - ${spark.version} - - - - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - - ${akka.group} - akka-actor_${scala.binary.version} - ${akka.version} - - ${akka.group} - akka-remote_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-slf4j_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-testkit_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-zeromq_${scala.binary.version} - ${akka.version} - - - ${akka.group} - akka-actor_${scala.binary.version} - - + ${project.groupId} + zeppelin-spark-dependencies + ${project.version} + provided - + org.apache.maven @@ -497,356 +206,6 @@ - - - spark-1.1 - - - - - 1.1.1 - 2.2.3-shaded-protobuf - - - - - cassandra-spark-1.1 - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - 1.1.1 - - - org.joda - joda-convert - - - - - - 1.1.1 - 2.2.3-shaded-protobuf - - - - - spark-1.2 - - - - 1.2.1 - - - - - cassandra-spark-1.2 - - 1.2.1 - - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - 1.2.1 - - - org.joda - joda-convert - - - - - - - - spark-1.3 - - - 1.3.1 - - - - - - - - - cassandra-spark-1.3 - - 1.3.0 - - - - - com.datastax.spark - spark-cassandra-connector_${scala.binary.version} - - 1.3.0-SNAPSHOT - - - org.joda - joda-convert - - - - - - - - spark-1.4 - - 1.4.1 - - - - - - - - hadoop-0.23 - - - - org.apache.avro - avro - - - - 0.23.10 - - - - - hadoop-1 - - 1.0.4 - hadoop1 - 1.8.8 - org.spark-project.akka - - - - - hadoop-2.2 - - 2.2.0 - 2.5.0 - hadoop2 - - - - - hadoop-2.3 - - 2.3.0 - 2.5.0 - 0.9.3 - hadoop2 - - - - - hadoop-2.4 - - 2.4.0 - 2.5.0 - 0.9.3 - hadoop2 - - - - - hadoop-2.6 - - 2.6.0 - 2.5.0 - 0.9.3 - hadoop2 - - - - - mapr3 - - false - - - 1.0.3-mapr-3.0.3 - 2.3.0-mapr-4.0.0-FCS - 0.7.1 - - - - - mapr4 - - false - - - 2.3.0-mapr-4.0.0-FCS - 2.3.0-mapr-4.0.0-FCS - 0.7.1 - - - - org.apache.curator - curator-recipes - 2.4.0 - - - org.apache.zookeeper - zookeeper - - - - - org.apache.zookeeper - zookeeper - 3.4.5-mapr-1406 - - - - - - yarn - - - org.apache.spark - spark-yarn_2.10 - ${spark.version} - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - - - - pyspark - - http://www.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz - - - - - - com.googlecode.maven-download-plugin - download-maven-plugin - 1.2.1 - - - download-pyspark-files - validate - - wget - - - ${spark.download.url} - true - ${project.build.directory}/spark-dist - - - - - - maven-clean-plugin - - - - ${basedir}/../python/build - - - ${project.build.directory}/spark-dist - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - download-and-zip-pyspark-files - generate-resources - - run - - - - - - - - - - - - - - - - - - hadoop-provided - - false - - - - org.apache.hadoop - hadoop-client - provided - - - org.apache.hadoop - hadoop-yarn-api - provided - - - org.apache.hadoop - hadoop-yarn-common - provided - - - org.apache.hadoop - hadoop-yarn-server-web-proxy - provided - - - org.apache.hadoop - hadoop-yarn-client - provided - - - org.apache.avro - avro - provided - - - org.apache.avro - avro-ipc - provided - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - provided - - - - - @@ -933,7 +292,6 @@ - org.apache.maven.plugins maven-dependency-plugin @@ -950,7 +308,7 @@ false false true - org.datanucleus + runtime @@ -963,6 +321,7 @@ false false true + runtime ${project.groupId} From 2052aa3d4f3183d2a88dda823bedcd51feceed46 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 10:59:47 -0700 Subject: [PATCH 02/32] Load interpreter/spark/dep only when SPARK_HOME is undefined --- bin/interpreter.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 93ae1e55088..e27b64d2af6 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -73,19 +73,23 @@ if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then $(mkdir -p "${ZEPPELIN_LOG_DIR}") fi -if [[ ! -z "${SPARK_HOME}" ]]; then - PYSPARKPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip" -else - PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip" -fi +# set spark related env variables +if [[ "${INTERPRETER_ID}" == "spark" ]]; then + if [[ -z "${SPARK_HOME}" ]]; then + PYSPARKPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip" + else + addJarInDir "${INTERPRETER_DIR}/dep" + PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip" + fi -if [[ x"" == x"${PYTHONPATH}" ]]; then - export PYTHONPATH="${PYSPARKPATH}" -else - export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}" -fi + if [[ x"" == x"${PYTHONPATH}" ]]; then + export PYTHONPATH="${PYSPARKPATH}" + else + export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}" + fi -unset PYSPARKPATH + unset PYSPARKPATH +fi ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} -cp ${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & pid=$! From c49be62f87df0c5d47ddc841f1fc8464c6764db0 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 11:38:16 -0700 Subject: [PATCH 03/32] Add hadoop jar and spark jar from HADOOP_HOME, SPARK_HOME when they are defined --- bin/common.sh | 14 +------------ bin/interpreter.sh | 50 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/bin/common.sh b/bin/common.sh index 7aab870af14..188ff863966 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -80,22 +80,10 @@ function addEachJarInDir(){ function addJarInDir(){ if [[ -d "${1}" ]]; then - export ZEPPELIN_CLASSPATH="${1}/*:${ZEPPELIN_CLASSPATH}" + ZEPPELIN_CLASSPATH="${1}/*:${ZEPPELIN_CLASSPATH}" fi } -if [[ ! -z "${SPARK_HOME}" ]] && [[ -d "${SPARK_HOME}" ]]; then - addJarInDir "${SPARK_HOME}" -fi - -if [[ ! -z "${HADOOP_HOME}" ]] && [[ -d "${HADOOP_HOME}" ]]; then - addJarInDir "${HADOOP_HOME}" -fi - -if [[ ! -z "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then - ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" -fi - export ZEPPELIN_CLASSPATH # Text encoding for diff --git a/bin/interpreter.sh b/bin/interpreter.sh index e27b64d2af6..93102c9d8c6 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -75,13 +75,61 @@ fi # set spark related env variables if [[ "${INTERPRETER_ID}" == "spark" ]]; then - if [[ -z "${SPARK_HOME}" ]]; then + # add Hadoop jars into classpath + if [[ ! -z "${HADOOP_HOME}" ]]; then + # Apache + addEachJarInDir "${HADOOP_HOME}/share" + + # CDH + addJarInDir "${HADOOP_HOME}" + addJarInDir "${HADOOP_HOME}/lib" + fi + + # autodetect HADOOP_CONF_HOME by heuristic + if [[ ! -z "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then + if [[ -d "${HADOOP_HOME}/etc/hadoop" ]]; then + HADOOP_CONF_DIR="${HADOOP_HOME}/etc/hadoop" + elif [[ -d "/etc/hadoop/conf" ]]; then + HADOOP_CONF_DIR="/etc/hadoop/conf" + fi + fi + + if [[ ! -z "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then + ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" + fi + + # add Spark jars into classpath + if [[ ! -z "${SPARK_HOME}" ]]; then + addJarInDir "${SPARK_HOME}/lib" PYSPARKPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip" else addJarInDir "${INTERPRETER_DIR}/dep" PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip" fi + # autodetect SPARK_CONF_DIR + if [[ ! -z "${SPARK_HOME}" ]] && [[ -z "${SPARK_CONF_DIR}" ]]; then + if [[ -d "${SPARK_HOME}/conf" ]]; then + SPARK_CONF_DIR="${SPARK_HOME}/conf" + fi + fi + + # read spark conf + if [[ -d "${SPARK_CONF_DIR}" ]]; then + while read line; do + echo "${line}" | grep -e "^spark[.]" > /dev/null + if [ $? -ne 0 ]; then + # skip the line not started with 'spark.' + continue; + fi + SPARK_CONF_KEY=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\1/g'` + SPARK_CONF_VALUE=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\2/g'` + export ZEPPELIN_JAVA_OPTS+=" -D${SPARK_CONF_KEY}=\"${SPARK_CONF_VALUE}\"" + done < ${SPARK_CONF_DIR} + fi + + export ZEPPELIN_CLASSPATH + if [[ x"" == x"${PYTHONPATH}" ]]; then export PYTHONPATH="${PYSPARKPATH}" else From b1d62a528b7ecb419cfcf12b81fd1620452890a0 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 11:52:24 -0700 Subject: [PATCH 04/32] Add scala-library in test scope --- spark/pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/spark/pom.xml b/spark/pom.xml index 1a43db5052e..72c8be1f002 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -34,8 +34,8 @@ Zeppelin spark support http://zeppelin.incubator.apache.org - + 2.10.4 2.10 @@ -192,6 +192,13 @@ + + org.scala-lang + scala-library + ${scala.version} + test + + org.scalatest scalatest_${scala.binary.version} From 1b7f95167641a7838d68334abfab4b6cfb0b8b90 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 16:26:30 -0700 Subject: [PATCH 05/32] Add dependency for compile and test --- spark/pom.xml | 126 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 35 deletions(-) diff --git a/spark/pom.xml b/spark/pom.xml index 72c8be1f002..26b3b7f8cb1 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -35,8 +35,12 @@ http://zeppelin.incubator.apache.org + 1.4.1 2.10.4 2.10 + + 2.3.0 + 0.8.2.1 @@ -70,6 +74,12 @@ ${project.version} provided + + + com.google.guava + guava + 14.0.1 + @@ -191,14 +201,93 @@ 1.1 - + + + org.apache.spark + spark-core_2.10 + ${spark.version} + provided + + + + org.apache.spark + spark-repl_2.10 + ${spark.version} + provided + + + + org.apache.spark + spark-sql_2.10 + ${spark.version} + provided + + + + org.apache.spark + spark-hive_2.10 + ${spark.version} + provided + + + + org.apache.spark + spark-catalyst_2.10 + ${spark.version} + provided + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + provided + + org.scala-lang scala-library ${scala.version} - test + provided + + + + org.scala-lang + scala-compiler + ${scala.version} + provided + + + + org.scala-lang + scala-reflect + ${scala.version} + provided + + + + commons-lang + commons-lang + provided + + org.apache.commons + commons-compress + 1.9 + provided + + + + net.sf.py4j + py4j + ${py4j.version} + + + + org.scalatest scalatest_${scala.binary.version} @@ -266,39 +355,6 @@ - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - - *:* - - org/datanucleus/** - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - reference.conf - - - - - - package - - shade - - - - - org.apache.maven.plugins maven-dependency-plugin From 0f9598b2b0c92cd4ea603a1e2e640bc550a930a9 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 16:26:50 -0700 Subject: [PATCH 06/32] py4j version as a property --- spark-dependencies/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index 1e48d6b4d85..9d10cf8a5ed 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -51,6 +51,7 @@ 2.3.4-spark http://www.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz + 0.8.2.1 @@ -632,7 +633,7 @@ + file="${project.build.directory}/spark-dist/spark-${spark.version}/python/lib/py4j-${py4j.version}-src.zip"/> From c3d96c18ffede93fa6ec43225ef5f1a1094a807d Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 16:27:11 -0700 Subject: [PATCH 07/32] Handle ZEPPELIN_CLASSPATH proper way --- bin/interpreter.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 93102c9d8c6..362302e5475 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -57,9 +57,6 @@ fi addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" addJarInDir "${INTERPRETER_DIR}" -export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" -CLASSPATH+=":${ZEPPELIN_CLASSPATH}" - HOSTNAME=$(hostname) ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer @@ -88,9 +85,9 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then # autodetect HADOOP_CONF_HOME by heuristic if [[ ! -z "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then if [[ -d "${HADOOP_HOME}/etc/hadoop" ]]; then - HADOOP_CONF_DIR="${HADOOP_HOME}/etc/hadoop" + export HADOOP_CONF_DIR="${HADOOP_HOME}/etc/hadoop" elif [[ -d "/etc/hadoop/conf" ]]; then - HADOOP_CONF_DIR="/etc/hadoop/conf" + export HADOOP_CONF_DIR="/etc/hadoop/conf" fi fi @@ -101,7 +98,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then # add Spark jars into classpath if [[ ! -z "${SPARK_HOME}" ]]; then addJarInDir "${SPARK_HOME}/lib" - PYSPARKPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip" + PYSPARKPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip" else addJarInDir "${INTERPRETER_DIR}/dep" PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip" @@ -128,8 +125,6 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then done < ${SPARK_CONF_DIR} fi - export ZEPPELIN_CLASSPATH - if [[ x"" == x"${PYTHONPATH}" ]]; then export PYTHONPATH="${PYSPARKPATH}" else @@ -139,6 +134,9 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then unset PYSPARKPATH fi +export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +CLASSPATH+=":${ZEPPELIN_CLASSPATH}" + ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} -cp ${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & pid=$! if [[ -z "${pid}" ]]; then From 9e812e7ef9a0c386aebaf332df736997269d67c1 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 18:36:44 -0700 Subject: [PATCH 08/32] Use reflection not to use import org.apache.spark.scheduler.Stage --- .../zeppelin/spark/SparkInterpreter.java | 81 ++++++----- .../zeppelin/spark/SparkSqlInterpreter.java | 126 +----------------- 2 files changed, 46 insertions(+), 161 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index a4ff494ce1c..f3a6a6b1291 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -29,6 +29,7 @@ import java.util.*; import com.google.common.base.Joiner; + import org.apache.spark.HttpServer; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; @@ -40,7 +41,6 @@ import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.DAGScheduler; import org.apache.spark.scheduler.Pool; -import org.apache.spark.scheduler.Stage; import org.apache.spark.sql.SQLContext; import org.apache.spark.ui.jobs.JobProgressListener; import org.apache.zeppelin.interpreter.Interpreter; @@ -67,6 +67,7 @@ import scala.collection.Iterator; import scala.collection.JavaConversions; import scala.collection.JavaConverters; +import scala.collection.Seq; import scala.collection.mutable.HashMap; import scala.collection.mutable.HashSet; import scala.tools.nsc.Settings; @@ -671,18 +672,26 @@ public int getProgress(InterpreterContext context) { if (jobGroup.equals(g)) { int[] progressInfo = null; - if (sc.version().startsWith("1.0")) { - progressInfo = getProgressFromStage_1_0x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.1")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.2")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.3")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.4")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else { - continue; + try { + Object finalStage = job.getClass().getMethod("finalStage").invoke(job); + if (sc.version().startsWith("1.0")) { + progressInfo = getProgressFromStage_1_0x(sparkListener, finalStage); + } else if (sc.version().startsWith("1.1")) { + progressInfo = getProgressFromStage_1_1x(sparkListener, finalStage); + } else if (sc.version().startsWith("1.2")) { + progressInfo = getProgressFromStage_1_1x(sparkListener, finalStage); + } else if (sc.version().startsWith("1.3")) { + progressInfo = getProgressFromStage_1_1x(sparkListener, finalStage); + } else if (sc.version().startsWith("1.4")) { + progressInfo = getProgressFromStage_1_1x(sparkListener, finalStage); + } else { + continue; + } + } catch (IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException + | SecurityException e) { + logger.error("Can't get progress info", e); + return 0; } totalTasks += progressInfo[0]; completedTasks += progressInfo[1]; @@ -695,33 +704,27 @@ public int getProgress(InterpreterContext context) { return completedTasks * 100 / totalTasks; } - private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Stage stage) { - int numTasks = stage.numTasks(); + private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Object stage) + throws IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { + int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); int completedTasks = 0; - Method method; + int id = (int) stage.getClass().getMethod("id").invoke(stage); + Object completedTaskInfo = null; - try { - method = sparkListener.getClass().getMethod("stageIdToTasksComplete"); - completedTaskInfo = - JavaConversions.asJavaMap((HashMap) method.invoke(sparkListener)).get( - stage.id()); - } catch (NoSuchMethodException | SecurityException e) { - logger.error("Error while getting progress", e); - } catch (IllegalAccessException e) { - logger.error("Error while getting progress", e); - } catch (IllegalArgumentException e) { - logger.error("Error while getting progress", e); - } catch (InvocationTargetException e) { - logger.error("Error while getting progress", e); - } + + completedTaskInfo = JavaConversions.asJavaMap( + (HashMap) sparkListener.getClass() + .getMethod("stageIdToTasksComplete").invoke(sparkListener)).get(id); if (completedTaskInfo != null) { completedTasks += (int) completedTaskInfo; } - List parents = JavaConversions.asJavaList(stage.parents()); + List parents = JavaConversions.asJavaList((Seq) stage.getClass() + .getMethod("parents").invoke(stage)); if (parents != null) { - for (Stage s : parents) { + for (Object s : parents) { int[] p = getProgressFromStage_1_0x(sparkListener, s); numTasks += p[0]; completedTasks += p[1]; @@ -731,9 +734,12 @@ private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Stage return new int[] {numTasks, completedTasks}; } - private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Stage stage) { - int numTasks = stage.numTasks(); + private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Object stage) + throws IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { + int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); int completedTasks = 0; + int id = (int) stage.getClass().getMethod("id").invoke(stage); try { Method stageIdToData = sparkListener.getClass().getMethod("stageIdToData"); @@ -747,7 +753,7 @@ private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Stage Set> keys = JavaConverters.asJavaSetConverter(stageIdData.keySet()).asJava(); for (Tuple2 k : keys) { - if (stage.id() == (int) k._1()) { + if (id == (int) k._1()) { Object uiData = stageIdData.get(k).get(); completedTasks += (int) numCompletedTasks.invoke(uiData); } @@ -756,9 +762,10 @@ private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Stage logger.error("Error on getting progress information", e); } - List parents = JavaConversions.asJavaList(stage.parents()); + List parents = JavaConversions.asJavaList((Seq) stage.getClass() + .getMethod("parents").invoke(stage)); if (parents != null) { - for (Stage s : parents) { + for (Object s : parents) { int[] p = getProgressFromStage_1_1x(sparkListener, s); numTasks += p[0]; completedTasks += p[1]; diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index d3bda44cf9d..9276ffe070c 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -17,25 +17,17 @@ package org.apache.zeppelin.spark; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.List; import java.util.Properties; -import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.spark.SparkContext; -import org.apache.spark.scheduler.ActiveJob; -import org.apache.spark.scheduler.DAGScheduler; -import org.apache.spark.scheduler.Stage; import org.apache.spark.sql.SQLContext; -import org.apache.spark.ui.jobs.JobProgressListener; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterUtils; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.WrappedInterpreter; @@ -44,13 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Tuple2; -import scala.collection.Iterator; -import scala.collection.JavaConversions; -import scala.collection.JavaConverters; -import scala.collection.mutable.HashMap; -import scala.collection.mutable.HashSet; - /** * Spark SQL interpreter for Zeppelin. * @@ -151,117 +136,10 @@ public FormType getFormType() { @Override public int getProgress(InterpreterContext context) { - String jobGroup = getJobGroup(context); - SQLContext sqlc = getSparkInterpreter().getSQLContext(); - SparkContext sc = sqlc.sparkContext(); - JobProgressListener sparkListener = getSparkInterpreter().getJobProgressListener(); - int completedTasks = 0; - int totalTasks = 0; - - DAGScheduler scheduler = sc.dagScheduler(); - HashSet jobs = scheduler.activeJobs(); - Iterator it = jobs.iterator(); - while (it.hasNext()) { - ActiveJob job = it.next(); - String g = (String) job.properties().get("spark.jobGroup.id"); - if (jobGroup.equals(g)) { - int[] progressInfo = null; - if (sc.version().startsWith("1.0")) { - progressInfo = getProgressFromStage_1_0x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.1")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.2")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.3")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else if (sc.version().startsWith("1.4")) { - progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); - } else { - logger.warn("Spark {} getting progress information not supported" + sc.version()); - continue; - } - totalTasks += progressInfo[0]; - completedTasks += progressInfo[1]; - } - } - - if (totalTasks == 0) { - return 0; - } - return completedTasks * 100 / totalTasks; - } - - private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Stage stage) { - int numTasks = stage.numTasks(); - int completedTasks = 0; - - Method method; - Object completedTaskInfo = null; - try { - method = sparkListener.getClass().getMethod("stageIdToTasksComplete"); - completedTaskInfo = - JavaConversions.asJavaMap((HashMap) method.invoke(sparkListener)).get( - stage.id()); - } catch (NoSuchMethodException | SecurityException e) { - logger.error("Error while getting progress", e); - } catch (IllegalAccessException e) { - logger.error("Error while getting progress", e); - } catch (IllegalArgumentException e) { - logger.error("Error while getting progress", e); - } catch (InvocationTargetException e) { - logger.error("Error while getting progress", e); - } - - if (completedTaskInfo != null) { - completedTasks += (int) completedTaskInfo; - } - List parents = JavaConversions.asJavaList(stage.parents()); - if (parents != null) { - for (Stage s : parents) { - int[] p = getProgressFromStage_1_0x(sparkListener, s); - numTasks += p[0]; - completedTasks += p[1]; - } - } - - return new int[] {numTasks, completedTasks}; + SparkInterpreter sparkInterpreter = getSparkInterpreter(); + return sparkInterpreter.getProgress(context); } - private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Stage stage) { - int numTasks = stage.numTasks(); - int completedTasks = 0; - - try { - Method stageIdToData = sparkListener.getClass().getMethod("stageIdToData"); - HashMap, Object> stageIdData = - (HashMap, Object>) stageIdToData.invoke(sparkListener); - Class stageUIDataClass = - this.getClass().forName("org.apache.spark.ui.jobs.UIData$StageUIData"); - - Method numCompletedTasks = stageUIDataClass.getMethod("numCompleteTasks"); - - Set> keys = - JavaConverters.asJavaSetConverter(stageIdData.keySet()).asJava(); - for (Tuple2 k : keys) { - if (stage.id() == (int) k._1()) { - Object uiData = stageIdData.get(k).get(); - completedTasks += (int) numCompletedTasks.invoke(uiData); - } - } - } catch (Exception e) { - logger.error("Error on getting progress information", e); - } - - List parents = JavaConversions.asJavaList(stage.parents()); - if (parents != null) { - for (Stage s : parents) { - int[] p = getProgressFromStage_1_1x(sparkListener, s); - numTasks += p[0]; - completedTasks += p[1]; - } - } - return new int[] {numTasks, completedTasks}; - } @Override public Scheduler getScheduler() { From f1e8789c2c3558422793b889ee03059de78ffb5a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 19:20:30 -0700 Subject: [PATCH 09/32] update travis config --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1b3c5dd7e1..114f723bae2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,22 +34,22 @@ script: - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.3.1 2.3 - - mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.2.1 2.3 - - mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.1.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From 2a61ecdc2bd27b9e7605e94278978e445352969c Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 20:21:51 -0700 Subject: [PATCH 10/32] Clear interpreter directory on mvn clean --- spark-dependencies/pom.xml | 11 +++++++++++ spark/pom.xml | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index 9d10cf8a5ed..c451c391cf7 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -689,6 +689,17 @@ + + maven-clean-plugin + + + + ../interpreter/spark/dep + + + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/spark/pom.xml b/spark/pom.xml index 26b3b7f8cb1..59da081e2e1 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -344,6 +344,17 @@ + + maven-clean-plugin + + + + ../interpreter/spark + + + + + org.apache.maven.plugins maven-surefire-plugin From 2ca3d9561c82ba51a27dd16965869d973d8b3b59 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 21 Aug 2015 21:39:01 -0700 Subject: [PATCH 11/32] set SPARK_HOME --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 114f723bae2..f25c4a758a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_install: - "sh -e /etc/init.d/xvfb start" install: - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B + - mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B before_script: - @@ -31,25 +31,25 @@ script: # spark 1.4 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - ./testing/startSparkCluster.sh 1.4.0 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B + - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.3.1 2.3 - - mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.2.1 2.3 - - mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.1.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3 /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From 9d6b40f068d5829ed4b8bbce122eaf79beb4bad4 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 07:19:31 -0700 Subject: [PATCH 12/32] Update .travis --- .travis.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index f25c4a758a7..9c38151fe53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,22 +34,19 @@ script: - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3 /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3 /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From df8f0ba5fecc92592e87f9e4a1b3228629c8eeb0 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 07:35:48 -0700 Subject: [PATCH 13/32] test more efficiently --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c38151fe53..9d1cc794db2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,17 +36,17 @@ script: # spark 1.3 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3 mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3 mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3 /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3 mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From 2606c0451f3e057fe051d340efc1becae3303104 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 08:05:11 -0700 Subject: [PATCH 14/32] bringing travis-install.sh back --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d1cc794db2..0fc30192c97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,22 +31,22 @@ script: # spark 1.4 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - ./testing/startSparkCluster.sh 1.4.0 2.3 - - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3 mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3 mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3 mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From cf0a61e7d984b09d233d517e6b4d47f06d318628 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 08:44:41 -0700 Subject: [PATCH 15/32] rm -rf only interpreter directory instead of mvn clean --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0fc30192c97..9dad087d4d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,17 +34,20 @@ script: - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - rm -rf `pwd`/interpreter/spark + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - rm -rf `pwd`/interpreter/spark + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - rm -rf `pwd`/interpreter/spark + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 From 8de7addbb73b6be1104482b493367fe3207e68ed Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 09:38:27 -0700 Subject: [PATCH 16/32] trying to find why travis is not closing the test --- .travis.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9dad087d4d8..8c6322485e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,23 +34,23 @@ script: - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.3.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.3.1 2.3 +# - rm -rf `pwd`/interpreter/spark +# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' +# - ./testing/startSparkCluster.sh 1.3.1 2.3 +# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false +# - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.2.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.2.1 2.3 +# - rm -rf `pwd`/interpreter/spark +# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' +# - ./testing/startSparkCluster.sh 1.2.1 2.3 +# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false +# - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.1.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.1.1 2.3 +# - rm -rf `pwd`/interpreter/spark +# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' +# - ./testing/startSparkCluster.sh 1.1.1 2.3 +# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false +# - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: - cat target/rat.txt From 797c0e27d094f3faa58f7bf1a24c08393528e95f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 10:09:34 -0700 Subject: [PATCH 17/32] enable 1.3.x test --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c6322485e8..a68cd6775c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,11 +34,11 @@ script: - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 -# - rm -rf `pwd`/interpreter/spark -# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' -# - ./testing/startSparkCluster.sh 1.3.1 2.3 -# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false -# - ./testing/stopSparkCluster.sh 1.3.1 2.3 + - rm -rf `pwd`/interpreter/spark + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - ./testing/startSparkCluster.sh 1.3.1 2.3 + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 # - rm -rf `pwd`/interpreter/spark # - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' From 6304180a6cf8c572351560cccf49c5c380f683d2 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 10:38:59 -0700 Subject: [PATCH 18/32] enable 1.2.x test --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a68cd6775c1..8ba69264eb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,12 +39,12 @@ script: - ./testing/startSparkCluster.sh 1.3.1 2.3 - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 -# spark 1.2 -# - rm -rf `pwd`/interpreter/spark -# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' -# - ./testing/startSparkCluster.sh 1.2.1 2.3 -# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false -# - ./testing/stopSparkCluster.sh 1.2.1 2.3 + # spark 1.2 + - rm -rf `pwd`/interpreter/spark + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - ./testing/startSparkCluster.sh 1.2.1 2.3 + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 # - rm -rf `pwd`/interpreter/spark # - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' From cd4519caf90e888b1c55f457e80c5a21a2f185b1 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 10:59:28 -0700 Subject: [PATCH 19/32] try sys.stdout.write instead of print --- .travis.yml | 10 +++++----- dev/travis/save-logs.py | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8ba69264eb2..5185ca95940 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,11 +46,11 @@ script: - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 -# - rm -rf `pwd`/interpreter/spark -# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' -# - ./testing/startSparkCluster.sh 1.1.1 2.3 -# - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false -# - ./testing/stopSparkCluster.sh 1.1.1 2.3 + - rm -rf `pwd`/interpreter/spark + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - ./testing/startSparkCluster.sh 1.1.1 2.3 + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: - cat target/rat.txt diff --git a/dev/travis/save-logs.py b/dev/travis/save-logs.py index 5f4ad28bade..d0480e8f456 100755 --- a/dev/travis/save-logs.py +++ b/dev/travis/save-logs.py @@ -42,8 +42,7 @@ def main(file, cmd): errcode = process.wait() diff = datetime.now() - start sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count)) - print - print cmd, "done", errcode + sys.stdout.write("\n" + str(cmd) + " done " + str(errcode) + "\n") return errcode if __name__ == "__main__": From a0150cf3073180fda41a73c62afaaf77b5635cec Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 11:32:45 -0700 Subject: [PATCH 20/32] not use travis-install for mvn test --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5185ca95940..fcc2f5638bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,19 +37,19 @@ script: - rm -rf `pwd`/interpreter/spark - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - rm -rf `pwd`/interpreter/spark - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - rm -rf `pwd`/interpreter/spark - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From 5f8a73416ebb2d46701871fa17a77ffd2c679f32 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 11:33:48 -0700 Subject: [PATCH 21/32] test -> package --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcc2f5638bd..1540d76ce25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,19 +37,19 @@ script: - rm -rf `pwd`/interpreter/spark - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - - mvn test -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - rm -rf `pwd`/interpreter/spark - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 - - mvn test -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - rm -rf `pwd`/interpreter/spark - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 - - mvn test -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From af7a92530ff54d2dcf486db863ffff1e4cf64bda Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 12:13:37 -0700 Subject: [PATCH 22/32] add pyspark flag --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1540d76ce25..7dfcd039ab3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,19 +35,19 @@ script: - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 - mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 - mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 From 5edb6fdba6c69c1bce61b9dbcd7d37c87b81b38f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 13:02:42 -0700 Subject: [PATCH 23/32] Use reflection to call addListener --- .../org/apache/zeppelin/spark/SparkInterpreter.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index f3a6a6b1291..d695b49e08f 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -157,7 +157,13 @@ public boolean isSparkContextInitialized() { private static JobProgressListener setupListeners(SparkContext context) { JobProgressListener pl = new JobProgressListener(context.getConf()); - context.listenerBus().addListener(pl); + try { + Method m = context.listenerBus().getClass().getMethod("addListener", pl.getClass()); + m.invoke(context.listenerBus(), pl); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + e.printStackTrace(); + } return pl; } @@ -272,7 +278,7 @@ public SparkContext createSparkContext() { } //TODO(jongyoul): Move these codes into PySparkInterpreter.java - + String pysparkBasePath = getSystemDefault("SPARK_HOME", "spark.home", null); File pysparkPath; if (null == pysparkBasePath) { @@ -607,7 +613,7 @@ public InterpreterResult interpretInput(String[] lines) { String incomplete = ""; for (int l = 0; l < linesToRun.length; l++) { - String s = linesToRun[l]; + String s = linesToRun[l]; // check if next line starts with "." (but not ".." or "./") it is treated as an invocation if (l + 1 < linesToRun.length) { String nextLine = linesToRun[l + 1].trim(); From 62b8c45cebf0f33526c5a0c1ca3205a2c3adb7e5 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 13:49:04 -0700 Subject: [PATCH 24/32] Print all logs --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7dfcd039ab3..a1b467d9971 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,25 +29,25 @@ before_script: script: # spark 1.4 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B + - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - ./testing/startSparkCluster.sh 1.4.0 2.3 - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B + - mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 - mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 - mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - rm -rf `pwd`/interpreter/spark - - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 - mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 From 0c28561f2c54cb21379acfd583626af87dc274aa Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 14:29:44 -0700 Subject: [PATCH 25/32] call listenerBus() using reflection --- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index d695b49e08f..1d2530fffb2 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -158,8 +158,9 @@ public boolean isSparkContextInitialized() { private static JobProgressListener setupListeners(SparkContext context) { JobProgressListener pl = new JobProgressListener(context.getConf()); try { - Method m = context.listenerBus().getClass().getMethod("addListener", pl.getClass()); - m.invoke(context.listenerBus(), pl); + Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context); + Method m = listenerBus.getClass().getMethod("addListener", pl.getClass()); + m.invoke(listenerBus, pl); } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { e.printStackTrace(); From 615c395c08bd4dd24d5838d3cb06d5e4c868843a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 15:10:15 -0700 Subject: [PATCH 26/32] get correct method --- .../main/java/org/apache/zeppelin/spark/SparkInterpreter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 1d2530fffb2..e684c521fde 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -41,6 +41,7 @@ import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.DAGScheduler; import org.apache.spark.scheduler.Pool; +import org.apache.spark.scheduler.SparkListener; import org.apache.spark.sql.SQLContext; import org.apache.spark.ui.jobs.JobProgressListener; import org.apache.zeppelin.interpreter.Interpreter; @@ -159,7 +160,7 @@ private static JobProgressListener setupListeners(SparkContext context) { JobProgressListener pl = new JobProgressListener(context.getConf()); try { Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context); - Method m = listenerBus.getClass().getMethod("addListener", pl.getClass()); + Method m = listenerBus.getClass().getMethod("addListener", SparkListener.class); m.invoke(listenerBus, pl); } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { From 5a17d9ca6c21200516df376909efc0db11653ced Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 16:13:17 -0700 Subject: [PATCH 27/32] Call sqlContext.sql using reflection --- .../apache/zeppelin/spark/SparkSqlInterpreter.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index 9276ffe070c..04bc09030f8 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -17,6 +17,8 @@ package org.apache.zeppelin.spark; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.List; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; @@ -114,8 +116,15 @@ public InterpreterResult interpret(String st, InterpreterContext context) { sc.setLocalProperty("spark.scheduler.pool", null); } + Object rdd = null; + try { + Method sqlMethod = sqlc.getClass().getMethod("sql", String.class); + rdd = sqlMethod.invoke(sqlc, st); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + throw new InterpreterException(e); + } - Object rdd = sqlc.sql(st); String msg = ZeppelinContext.showDF(sc, context, rdd, maxResult); return new InterpreterResult(Code.SUCCESS, msg); } From 3a88c77ab80b9e118d09b781296753854862758b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 16:13:36 -0700 Subject: [PATCH 28/32] Test use explicitly %spark --- .../zeppelin/rest/AbstractTestRestApi.java | 16 ++++++++++++---- .../zeppelin/rest/ZeppelinSparkClusterTest.java | 12 ++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index aa395aaeb2b..aab80432cb8 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -108,8 +108,12 @@ protected static void startUp() throws Exception { // ci environment runs spark cluster for testing // so configure zeppelin use spark cluster if ("true".equals(System.getenv("CI"))) { - // assume first one is spark - InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0); + InterpreterSetting sparkIntpSetting = null; + for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) { + if (intpSetting.getGroup().equals("spark")) { + sparkIntpSetting = intpSetting; + } + } // set spark master sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); @@ -120,8 +124,12 @@ protected static void startUp() throws Exception { ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.id()); } else { - // assume first one is spark - InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0); + InterpreterSetting sparkIntpSetting = null; + for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) { + if (intpSetting.getGroup().equals("spark")) { + sparkIntpSetting = intpSetting; + } + } String sparkHome = getSparkHome(); if (sparkHome != null) { diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index fd4a8b301bf..aa2476a5d5d 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -64,7 +64,7 @@ public void basicRDDTransformationAndActionTest() throws IOException { // run markdown paragraph, again Paragraph p = note.addParagraph(); - p.setText("print(sc.parallelize(1 to 10).reduce(_ + _))"); + p.setText("%spark print(sc.parallelize(1 to 10).reduce(_ + _))"); note.run(p.getId()); waitForFinish(p); assertEquals("55", p.getResult().message()); @@ -88,7 +88,7 @@ public void pySparkTest() throws IOException { } ZeppelinServer.notebook.removeNote(note.id()); } - + @Test public void pySparkAutoConvertOptionTest() throws IOException { // create new note @@ -113,11 +113,11 @@ public void zRunTest() throws IOException { // create new note Note note = ZeppelinServer.notebook.createNote(); Paragraph p0 = note.addParagraph(); - p0.setText("z.run(1)"); + p0.setText("%spark z.run(1)"); Paragraph p1 = note.addParagraph(); - p1.setText("val a=10"); + p1.setText("%spark val a=10"); Paragraph p2 = note.addParagraph(); - p2.setText("print(a)"); + p2.setText("%spark print(a)"); note.run(p0.getId()); waitForFinish(p0); @@ -135,7 +135,7 @@ public void zRunTest() throws IOException { */ private int getSparkVersionNumber(Note note) { Paragraph p = note.addParagraph(); - p.setText("print(sc.version)"); + p.setText("%spark print(sc.version)"); note.run(p.getId()); waitForFinish(p); String sparkVersion = p.getResult().message(); From bacfd93bdc10001e0ea545c40f7cc7eb59038d37 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 17:00:15 -0700 Subject: [PATCH 29/32] Update readme --- README.md | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d5658587f63..a3bc0ad1ce8 100644 --- a/README.md +++ b/README.md @@ -38,50 +38,52 @@ sudo apt-get install npm ### Build If you want to build Zeppelin from the source, please first clone this repository. And then: ``` -mvn clean package +mvn clean package -DskipTests ``` -Build with specific version -Spark 1.1.x -``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests -``` -Spark 1.2.x +Build with specific Spark version + +Spark 1.4.x ``` -mvn clean package -Pspark-1.2 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests +mvn clean package -Pspark-1.4 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests ``` Spark 1.3.x ``` mvn clean package -Pspark-1.3 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests ``` -Spark 1.4.x +Spark 1.2.x ``` -mvn clean package -Pspark-1.4 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests +mvn clean package -Pspark-1.2 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests +``` +Spark 1.1.x +``` +mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests ``` CDH 5.X ``` mvn clean package -Pspark-1.2 -Dhadoop.version=2.5.0-cdh5.3.0 -Phadoop-2.4 -DskipTests ``` -Yarn (Hadoop 2.2.x) +Yarn (Hadoop 2.7.x) ``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -Pyarn -DskipTests +mvn clean package -Pspark-1.4 -Dspark.version=1.4.1 -Dhadoop.version=2.7.0 -Phadoop-2.6 -Pyarn -DskipTests ``` -Yarn (Hadoop 2.3.x) +Yarn (Hadoop 2.6.x) ``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.3.0 -Phadoop-2.3 -Pyarn -DskipTests +mvn clean package -Pspark-1.1 -Dhadoop.version=2.6.0 -Phadoop-2.6 -Pyarn -DskipTests ``` Yarn (Hadoop 2.4.x) ``` mvn clean package -Pspark-1.1 -Dhadoop.version=2.4.0 -Phadoop-2.4 -Pyarn -DskipTests ``` -Yarn (Hadoop 2.6.x) +Yarn (Hadoop 2.3.x) ``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.6.0 -Phadoop-2.6 -Pyarn -DskipTests +mvn clean package -Pspark-1.1 -Dhadoop.version=2.3.0 -Phadoop-2.3 -Pyarn -DskipTests ``` -Yarn (Hadoop 2.7.x) +Yarn (Hadoop 2.2.x) ``` -mvn clean package -Pspark-1.4 -Dspark.version=1.4.1 -Dhadoop.version=2.7.0 -Phadoop-2.6 -Pyarn -DskipTests +mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -Pyarn -DskipTests ``` + Ignite (1.1.0-incubating and later) ``` mvn clean package -Dignite.version=1.1.0-incubating -DskipTests @@ -96,6 +98,19 @@ If you wish to configure Zeppelin option (like port number), configure the follo (You can copy ```./conf/zeppelin-env.sh.template``` into ```./conf/zeppelin-env.sh```. Same for ```zeppelin-site.xml```.) + +#### Setting SPARK_HOME and HADOOP_HOME + +Without SPARK_HOME and HADOOP_HOME, Zeppelin uses embedded Spark and Hadoop binaries that you have specified with mvn build option. +If you want to use system provided Spark and Hadoop, export SPARK_HOME and HADOOP_HOME in zeppelin-env.sh +You can use any supported version of spark without rebuilding Zeppelin. + +``` +# ./conf/zeppelin-env.sh +export SPARK_HOME=... +export HADOOP_HOME=... +``` + #### External cluster configuration Mesos From eb4ec09a709a832b5ce9807ccee8510521cedad5 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 22 Aug 2015 18:15:46 -0700 Subject: [PATCH 30/32] fix reading spark-*.conf file --- bin/interpreter.sh | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 362302e5475..43ca3ad9887 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -111,18 +111,23 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi fi - # read spark conf + # read spark-*.conf if exists if [[ -d "${SPARK_CONF_DIR}" ]]; then - while read line; do - echo "${line}" | grep -e "^spark[.]" > /dev/null - if [ $? -ne 0 ]; then - # skip the line not started with 'spark.' - continue; - fi - SPARK_CONF_KEY=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\1/g'` - SPARK_CONF_VALUE=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\2/g'` - export ZEPPELIN_JAVA_OPTS+=" -D${SPARK_CONF_KEY}=\"${SPARK_CONF_VALUE}\"" - done < ${SPARK_CONF_DIR} + ls ${SPARK_CONF_DIR}/spark-*.conf > /dev/null 2>&1 + if [[ $? -eq 0 ]]; then + for file in ${SPARK_CONF_DIR}/spark-*.conf; do + while read -r line; do + echo "${line}" | grep -e "^spark[.]" > /dev/null + if [ $? -ne 0 ]; then + # skip the line not started with 'spark.' + continue; + fi + SPARK_CONF_KEY=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\1/g'` + SPARK_CONF_VALUE=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\2/g'` + export ZEPPELIN_JAVA_OPTS+=" -D${SPARK_CONF_KEY}=\"${SPARK_CONF_VALUE}\"" + done < "${file}" + done + fi fi if [[ x"" == x"${PYTHONPATH}" ]]; then From 57b3f9612ceff27fe6ea548539ed6f5ccc405b39 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 25 Aug 2015 12:06:31 -0700 Subject: [PATCH 31/32] Add comment --- .../java/org/apache/zeppelin/spark/SparkSqlInterpreter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index 04bc09030f8..053b887adc2 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -118,6 +118,10 @@ public InterpreterResult interpret(String st, InterpreterContext context) { Object rdd = null; try { + // method signature of sqlc.sql() is changed + // from def sql(sqlText: String): SchemaRDD (1.2 and prior) + // to def sql(sqlText: String): DataFrame (1.3 and later). + // Therefore need to use reflection to keep binary compatibility for all spark versions. Method sqlMethod = sqlc.getClass().getMethod("sql", String.class); rdd = sqlMethod.invoke(sqlc, st); } catch (NoSuchMethodException | SecurityException | IllegalAccessException From 654c37861e8d64862b3e4df25611a5f8541e8670 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 31 Aug 2015 12:12:54 -0700 Subject: [PATCH 32/32] use consistant, simpler expressions --- bin/interpreter.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 43ca3ad9887..61dd249d768 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -73,7 +73,7 @@ fi # set spark related env variables if [[ "${INTERPRETER_ID}" == "spark" ]]; then # add Hadoop jars into classpath - if [[ ! -z "${HADOOP_HOME}" ]]; then + if [[ -n "${HADOOP_HOME}" ]]; then # Apache addEachJarInDir "${HADOOP_HOME}/share" @@ -83,7 +83,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi # autodetect HADOOP_CONF_HOME by heuristic - if [[ ! -z "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then + if [[ -n "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then if [[ -d "${HADOOP_HOME}/etc/hadoop" ]]; then export HADOOP_CONF_DIR="${HADOOP_HOME}/etc/hadoop" elif [[ -d "/etc/hadoop/conf" ]]; then @@ -91,12 +91,12 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi fi - if [[ ! -z "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then + if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" fi # add Spark jars into classpath - if [[ ! -z "${SPARK_HOME}" ]]; then + if [[ -n "${SPARK_HOME}" ]]; then addJarInDir "${SPARK_HOME}/lib" PYSPARKPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip" else @@ -105,7 +105,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi # autodetect SPARK_CONF_DIR - if [[ ! -z "${SPARK_HOME}" ]] && [[ -z "${SPARK_CONF_DIR}" ]]; then + if [[ -n "${SPARK_HOME}" ]] && [[ -z "${SPARK_CONF_DIR}" ]]; then if [[ -d "${SPARK_HOME}/conf" ]]; then SPARK_CONF_DIR="${SPARK_HOME}/conf" fi @@ -114,11 +114,11 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then # read spark-*.conf if exists if [[ -d "${SPARK_CONF_DIR}" ]]; then ls ${SPARK_CONF_DIR}/spark-*.conf > /dev/null 2>&1 - if [[ $? -eq 0 ]]; then + if [[ "$?" -eq 0 ]]; then for file in ${SPARK_CONF_DIR}/spark-*.conf; do while read -r line; do echo "${line}" | grep -e "^spark[.]" > /dev/null - if [ $? -ne 0 ]; then + if [ "$?" -ne 0 ]; then # skip the line not started with 'spark.' continue; fi @@ -130,7 +130,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi fi - if [[ x"" == x"${PYTHONPATH}" ]]; then + if [[ -z "${PYTHONPATH}" ]]; then export PYTHONPATH="${PYSPARKPATH}" else export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}"