python+spark 2.0+hadoop 机器学习与大数据实战 第八章代码

8.7

 HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/ pyspark --master yarn --deploy-mode client

textfile=sc.textFile("hdfs://master:9000/user/hduser/wordcount/input/LICENSE.txt")
textfile.count()
exit

cp /usr/local/spark/conf/spark-env.sh.template /usr/local/spark/conf/spark-env.sh
sudo gedit /usr/local/spark/conf/spark-env.sh
 

export SPARK_MASTER_IP=master
export SPARK_WORKER_CORES=1
export SPARK_WORKER_MEMORY=500m  (最少480m,有设定)
export SPARK_WORKER_INSTANCES=3

sssh data1
sudo mkdir /usr/local/spark
sudo chown hduser:hduser /usr/local/spark/
exit

sudo scp -r /usr/local/spark hduser@data1:/usr/local
 

/usr/local/spark/sbin/start-all.sh
pyspark --master spark://master:7077 --num-executors 1 --total-executor-cores 3 --executor-memory 500m
 

bash Anaconda2-2.5.0-Linux-x86_64.sh -b

export PATH=/home/hduser/anaconda2/bin:$PATH
export ANACONDA_PATH=/home/hduser/anaconda2
export PYSPARK_DRIVER_PYTHON=$ANACONDA_PATH/bin/ipython
export PYSPARK_PYTHON=$ANACONDA_PATH/bin/python

 

sudo gedit ~/.bashrc

source ~/.bashrc

python --version

mkdir -p ~/pythonwork/ipynotebook
cd ~/pythonwork/ipynotebook/
 

PYSPARK_DRIVER_PYTHON=ipython PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark    （打开jupyter notebook）