From 7a30a14fb2b3c0e41017d909ba5c1e1a0b9c545b Mon Sep 17 00:00:00 2001 From: Felix Cheung Date: Wed, 25 Mar 2015 11:15:53 -0700 Subject: [PATCH 1/2] minor doc update for running on YARN --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4ae849921e4..71dc92824a6 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,11 @@ If you set `SPARK_HOME`, you should deploy spark binary on the same location to Yarn # ./conf/zeppelin-env.sh - export export SPARK_YARN_JAR=/path/to/spark-assembly-*.jar + export SPARK_YARN_JAR=/path/to/spark-assembly-*.jar export HADOOP_CONF_DIR=/path/to/hadoop_conf_dir -`SPARK_YARN_JAR` is deployed for running executor. `HADOOP_CONF_DIR` should contains yarn-site.xml and core-site.xml. +`SPARK_YARN_JAR` is deployed for running executor, this could be a local path or HDFS. HDFS allows YARN to cache it on nodes so that it doesn't need to be distributed each time an application runs. To point to a jar on HDFS, for example, set this configuration to "hdfs:///some/path". +`HADOOP_CONF_DIR` should contains yarn-site.xml and core-site.xml. ### Run ./bin/zeppelin-daemon.sh start From e89ba083c21847a0b99c07735f106561ceee122b Mon Sep 17 00:00:00 2001 From: Felix Cheung Date: Tue, 31 Mar 2015 13:24:16 -0700 Subject: [PATCH 2/2] PySpark Interpreter should allow starting with a specific version of Python, as PySpark does. --- .../java/com/nflabs/zeppelin/spark/PySparkInterpreter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java index 071fceac7b8..36cdcaeb432 100644 --- a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java @@ -64,7 +64,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand new InterpreterPropertyBuilder() .add("spark.home", SparkInterpreter.getSystemDefault("SPARK_HOME", "spark.home", ""), - "Spark home path. Should be provided for pyspark").build()); + "Spark home path. Should be provided for pyspark") + .add("zeppelin.pyspark.python", + SparkInterpreter.getSystemDefault("PYSPARK_PYTHON", null, "python"), + "Python command to run pyspark with").build()); } public PySparkInterpreter(Properties property) { @@ -115,7 +118,7 @@ public void open() { gatewayServer.start(); // Run python shell - CommandLine cmd = CommandLine.parse("python"); + CommandLine cmd = CommandLine.parse(getProperty("zeppelin.pyspark.python")); cmd.addArgument(scriptPath, false); cmd.addArgument(Integer.toString(port), false); executor = new DefaultExecutor();