~/.bashrc
export JAVA_HOME=/usr/java/latest/
export HADOOP_HOME=~/hadoop
export SPARK_HOME=~/spark
export PATH=.:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
export SPARK_MASTER_HOST=192.168.202.134
export SPARK_LOCAL_IP=192.168.202.134
spark/conf/spark-env.sh
export SPARK_LOCAL_IP=192.168.202.134
export SPARK_MASTER_HOST=192.168.202.134
Start spark master and slaves.
sh ./start-master.sh
sh ./ start-slaves.sh
Submit spark task:
spark/bin/spark-submit --class SparkTest --master spark://192.168.202.134:7077 --deploy-mode cluster -
-supervise --total-executor-cores 2 andy.jar
Submit spark task on Windows, got the following error:
17/03/10 15:52:16 ERROR ClientEndpoint: Exception from cluster was: java.nio.file.NoSuchFileException: /D:/Tools/spark-2.1.0-bin-hadoop2.7/bin/andy.jar
java.nio.file.NoSuchFileException: /D:/Tools/spark-2.1.0-bin-hadoop2.7/bin/andy.jar
export JAVA_HOME=/usr/java/latest/
export HADOOP_HOME=~/hadoop
export SPARK_HOME=~/spark
export PATH=.:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
export SPARK_MASTER_HOST=192.168.202.134
export SPARK_LOCAL_IP=192.168.202.134
spark/conf/spark-env.sh
export SPARK_LOCAL_IP=192.168.202.134
export SPARK_MASTER_HOST=192.168.202.134
Start spark master and slaves.
sh ./start-master.sh
sh ./ start-slaves.sh
Submit spark task:
spark/bin/spark-submit --class SparkTest --master spark://192.168.202.134:7077 --deploy-mode cluster -
-supervise --total-executor-cores 2 andy.jar
Submit spark task on Windows, got the following error:
17/03/10 15:52:16 ERROR ClientEndpoint: Exception from cluster was: java.nio.file.NoSuchFileException: /D:/Tools/spark-2.1.0-bin-hadoop2.7/bin/andy.jar
java.nio.file.NoSuchFileException: /D:/Tools/spark-2.1.0-bin-hadoop2.7/bin/andy.jar
来自 “ ITPUB博客 ” ,链接:http://blog.itpub.net/10742815/viewspace-2135111/,如需转载,请注明出处,否则将追究法律责任。
转载于:http://blog.itpub.net/10742815/viewspace-2135111/