大数据（1d）集群脚本

先配置好集群免密登录
https://yellow520.blog.youkuaiyun.com/article/details/110143502

0、家目录下创建bin目录

mkdir ~/bin
cd ~/bin

1、集群文件同路径传输

touch rsync.py
chmod 777 rsync.py
vim rsync.py

#!/usr/bin/python2
# coding:utf-8
import os, re, socket

# 获取第一个参数
a = os.sys.argv
if len(a) != 2:
    print a
    exit('使用方法：rsync.py $HADOOP_HOME')
a = a[1]

# 补全路径
if not a.startswith('/'):
    a = os.path.join(os.getcwd(), a)
if os.path.isdir(a) and not a.endswith('/'):
    a += '/'
if not os.path.exists(a):
    print a
    exit('路径不存在')

# 文件分发
hostname = socket.gethostname()
with open('/etc/hosts') as f:
    for i in set(re.findall('hadoop[0-9]+', f.read())):
        if i == hostname:
            continue
        aa = 'rsync -a %s %s:%s' % (a, i, a)
        print aa
        os.system(aa)

分发

rsync.py ~/bin

2、集群操作脚本

touch ~/bin/cluster.py
chmod 777 ~/bin/cluster.py
vim ~/bin/cluster.py

#!/usr/bin/python2
import os, re

# command
a = os.sys.argv
if len(a) < 2:
    print a
    exit('Usage: cluster.py python --version')
cmd = ' '.join(a[1:])

# cluster operation
with open('/etc/hosts') as f:
    for h in sorted(set(re.findall('hadoop[0-9]+', f.read()))):
        c = "ssh %s '%s'" % (h, cmd)
        print '\033[033m{}\033[0m'.format(c)
        os.system(c)

3、集群jps

touch ~/bin/jps.py
chmod 777 ~/bin/jps.py
vim ~/bin/jps.py

然后写入

#!/usr/bin/python2
import os, re
with open('/etc/hosts') as f:
    for i in sorted(set(re.findall('hadoop[0-9]+', f.read()))):
        print i.center(40, '-')
        os.system('ssh %s "jps | grep -v Jps"' % i)

4、集群source环境变量

https://yellow520.blog.youkuaiyun.com/article/details/112692486

5、集群启停

下面是简单版，另有集群启停脚本优化版

5.1、ZooKeeper群起

touch $ZOOKEEPER_HOME/bin/zookeeper.py
chmod 777 $ZOOKEEPER_HOME/bin/zookeeper.py
vim $ZOOKEEPER_HOME/bin/zookeeper.py

#!/usr/bin/python2
import os, re

CMD = 'zkServer.sh '
option = 'status'

a = os.sys.argv
if len(a) == 2:
    a = a[1]
    if a in {'stop', 'start', 'restart'}:
        option = a
cmd = CMD + option

with open('/etc/hosts') as f:
    for h in set(re.findall('hadoop[0-9]+', f.read())):
        _cmd = 'ssh %s "%s"' % (h, cmd)
        print(_cmd)
        os.system(_cmd)

5.2、Kafka群起

touch $KAFKA_HOME/bin/kafka.py
chmod 777 $KAFKA_HOME/bin/kafka.py
vim $KAFKA_HOME/bin/kafka.py

#!/usr/bin/python2
import os, re, socket

PATTERN = 'hadoop[0-9]+'

cmd = 'kafka-topics.sh --zookeeper {}:2181/kafka --list'.format
commands = {
    'stop': 'kafka-server-stop.sh',
    'start': 'kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties'
}

a = os.sys.argv
if (len(a) == 2) and (a[1] in commands):
    cmd = commands[a[1]]
    with open('/etc/hosts') as f:
        for i in set(re.findall(PATTERN, f.read())):
            os.system('ssh %s "%s"' % (i, cmd))
else:
    hostname = socket.gethostname()
    os.system('ssh %s "%s"' % (hostname, cmd(hostname)))

5.3、高可用Hadoop集群启停

touch $HADOOP_HOME/sbin/haHadoop.py
chmod 777 $HADOOP_HOME/sbin/haHadoop.py
vim $HADOOP_HOME/sbin/haHadoop.py

#!/usr/bin/python2
import os, re

def state():
    os.system('hdfs fsck /')

def start():
    os.system('start-dfs.sh')
    os.system('start-yarn.sh')

def stop():
    os.system('stop-yarn.sh')
    os.system('stop-dfs.sh')

def restart():
    stop()
    start()

a = os.sys.argv
if len(a) == 2 and a[1] in {'start', 'stop', 'restart'}:
    exec(a[1] + '()')
else:
    state()

5.4、HIVE启停

https://yellow520.blog.youkuaiyun.com/article/details/115944692

6、集群Linux文件删除

• 功能：删除集群中每台机器的某个文件
• 链接：https://yellow520.blog.youkuaiyun.com/article/details/115725340

7、虚拟机集群关机

touch /usr/sbin/init0.py
chmod 777 /usr/sbin/init0.py
vim /usr/sbin/init0.py

#!/usr/bin/python2
import os, re, socket
hostname = socket.gethostname()
with open('/etc/hosts') as f:
    for h in set(re.findall('hadoop[0-9]+', f.read())):
        if h != hostname:
            os.system('ssh %s "init 0"' % h)
os.system('init 0')