Python学习

 1、Python脚本中调用shell命令

result=subprocess.call(["curl -k --get --data \'session.id={}&ajax=executeFlow&project={}&flow={}\' {}".format("d1d0073b-1447-4138-8c58-d30544f11681","test","insp_big_data",azkabanURL)],shell=True)

#call返回值只有0和1，int类型
#check_output返回值为byte类型
将byte类型转换为字符串
s = str(check_output_value,'utf-8')
j = eval(s)
value = j[key]

#json格式化
result=requests.get(url).json()
print (json.dumps(result,indent=2,ensure_ascii=False))

2、安装pip

#wget --no-check-certificate https://bootstrap.pypa.io/ez_setup.py

#sudo python ez_setup.py --insecure

#wget https://pypi.python.org/packages/11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/pip-9.0.1.tar.gz#md5=35f01da33009719497f01a4ba69d63c9

#tar -xf pip-9.0.1.tar.gz&cd pip-9.0.1

#sudo python setup.py install

#ln -s /usr/local/python27/bin/pip /usr/bin/pip

3、安装Python

#Python2或Python3
#安装依赖
sudo yum install gcc make cmake zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel

#下载安装包（地址：https://www.python.org/ftp/python/）
wget https://www.python.org/ftp/python/2.5.4/Python-2.5.4.tgz

#解压，编译
./configure --with-ssl --prefix=/path/to/install
make 
make install

#建立软连接
ln -s /usr/local/bin/python2.5 /usr/bin/python


#echo "export PYSPARK_PYTHON=/usr/bin/python3" | sudo tee -a /etc/bashrc
#echo "source /etc/spark/conf/spark-env.sh" |sudo tee -a /etc/bashrc
echo "export PYSPARK_PYTHON=/usr/bin/python3" | sudo tee -a /etc/profile
echo "source /etc/spark/conf/spark-env.sh" |sudo tee -a /etc/profile
cat << EOF > ~/requirement.text
Cython>= 0.28.5
numpy==1.14.6
pandas==1.0.3
py4j==0.10.7
pyarrow==0.14.1
PyMySQL==0.9.3
# pypandoc==1.5
pyspark==2.4.5
python-dateutil==2.8.1
pytz==2019.3
six==1.14.0
SQLAlchemy==1.3.16
scipy==1.5.4
tensorflow==1.14.0
pywavelets
sklearn
EOF
sudo easy_install-3.6 pip && \
sudo /usr/local/bin/pip3 install wheel==0.34.2  pypandoc==1.5 -i  https://mirrors.cloud.tencent.com/pypi/simple &&  \
sudo  /usr/local/bin/pip3 install -r ~/requirement.text  -i  https://mirrors.cloud.tencent.com/pypi/simple

4、其他库安装

#离线安装boto3
#下载
pip3 download 库名 -d /tmp/boto3 
#安装
sudo pip3 install --no-index --find-links=/tmp/boto3/ boto3

5、pyspark

#cdh spark使用pyspark
#需要导入环境变量/etc/profile
export PYSPARK_PYTHON=/usr/bin/python3  #用于确定pyspark使用哪个python
source /etc/spark/conf/spark-env.sh  #用于确定使用的spark框架
#如果使用了apache的spark，执行source /etc/spark/conf/spark-env.sh即可

#root用户执行时依赖库缺失问题
import sys
sys.path.append('/home/username/.local/lib/python3.6/site-packages')

6、jupyter

#下载jupyter 
/usr/local/bin/pip3 install jupyter -i  http://mirrors.sxc.sh/pypi/simple/ --trusted-host mirrors.sxc.sh
#创建软连接
ln -s /usr/local/bin/jupyter /usr/bin/jupyter
#生成配置文件
jupyter notebook --generate-config
#编辑配置文件
vim /root/.jupyter/jupyter_notebook_config.py

#添加以下信息
c.NotebookApp.ip = '0.0.0.0'
c.NotebookApp.allow.root = True
c.NotebookApp.open_browser = False
c.NotebookApp.password = 'argon2:$argon2id$v=19$m=10240,t=10,p=8$Wsxa9+yZl1mEOG2iGKPcKg$vjkDV6vjEDdDCOd1YSiUAFcji+ewTP+s6I1TAMdonRU'
c.NotebookApp.port = 8890

#启动jupyter
nohup jupyter notebook --allow-root > /var/log/jupyter.log 2>&1 &

#密码：admin