大数据（1e）集群启停脚本优化（不定更）

非高可用Hadoop集群启停

touch $HADOOP_HOME/sbin/hadoop.py
chmod 777 $HADOOP_HOME/sbin/hadoop.py
vim $HADOOP_HOME/sbin/hadoop.py

#!/usr/bin/python2
# coding=utf-8
import os, re, subprocess, time

# HDFS
NN = re.compile(r'<name>fs\.defaultFS</name>\s*<value>hdfs://(hadoop\d+):\d+</value>')
NN_F = subprocess.check_output('cat $HADOOP_HOME/etc/hadoop/core-site.xml', shell=True)
nn = NN.findall(NN_F)[0]
start_dfs = 'ssh %s start-dfs.sh' % nn
stop_dfs = 'ssh %s stop-dfs.sh' % nn

# YARN
RM = re.compile(r'<name>yarn\.resourcemanager\.hostname</name>\s*<value>(hadoop\d+)</value>')
RM_F = subprocess.check_output('cat $HADOOP_HOME/etc/hadoop/yarn-site.xml', shell=True)
rm = RM.findall(RM_F)[0]
start_yarn = 'ssh %s start-yarn.sh' % rm
stop_yarn = 'ssh %s stop-yarn.sh' % rm

# history server
MR_F = subprocess.check_output('cat $HADOOP_HOME/etc/hadoop/mapred-site.xml', shell=True)
START_HISTORY_SERVER = 'mapred --daemon start historyserver'
STOP_HISTORY_SERVER = 'mapred --daemon stop historyserver'
HISTORY = re.compile(r'<name>mapreduce\.jobhistory\.address</name>\s*<value>(hadoop\d+):\d+</value>')

# 脚本参数选项
options = {
    'start', 'stop', 'restart', 'start_history_server', 'stop_history_server',
    'start_all', 'stop_all', 'restart_all'}

def execute(cmd):
    print(cmd)
    os.system(cmd)

def start():
    """启动HDFS和YARN"""
    execute(start_dfs)
    execute(start_yarn)

def stop():
    """停止HDFS和YARN"""
    execute(stop_dfs)
    execute(stop_yarn)

def restart():
    """重启HDFS和YARN"""
    stop()
    time.sleep(9)
    start()

def get_history_address():
    address = HISTORY.findall(MR_F)
    if address:
        return address[0]
    else:
        print('没有配置历史服务器')

def start_history_server():
    """启动历史服务器"""
    host = get_history_address()
    if host:
        execute("ssh {} '{}'".format(host, START_HISTORY_SERVER))

def stop_history_server():
    """停止历史服务器"""
    host = get_history_address()
    if host:
        execute("ssh {} '{}'".format(host, STOP_HISTORY_SERVER))

def start_all():
    """启动HDFS、YARN和历史服务器"""
    start()
    start_history_server()

def stop_all():
    """停止HDFS、YARN和历史服务器"""
    stop_history_server()
    stop()

def restart_all():
    """重启HDFS、YARN和历史服务器"""
    stop_history_server()
    restart()
    start_history_server()

def usage():
    print('Hadoop命令'.center(40, '='))
    print(start_dfs)
    print(start_yarn)
    print(stop_dfs)
    print(stop_yarn)
    print(START_HISTORY_SERVER)
    print(STOP_HISTORY_SERVER)
    print('脚本用法'.center(40, '='))
    for option in options:
        print('{} {} -> {}'.format(__file__, option, eval(option + '.__doc__')))

a = os.sys.argv
if len(a) == 2 and a[1] in options:
    exec(a[1] + '()')
else:
    usage()

高可用Hadoop集群启停

touch $HADOOP_HOME/sbin/haHadoop.py
chmod 777 $HADOOP_HOME/sbin/haHadoop.py
vim $HADOOP_HOME/sbin/haHadoop.py

#!/usr/bin/python2
import os, re, subprocess, time

def state():
    NN = re.compile(r'<name>dfs.ha.namenodes.+?</name>\s*<value>(.+?)</value>')
    RM = re.compile('<name>yarn.resourcemanager.ha.rm-ids</name>\s*<value>(.+?)</value>')
    NN_LS = subprocess.check_output('cat $HADOOP_HOME/etc/hadoop/hdfs-site.xml', shell=True)
    RM_LS = subprocess.check_output('cat $HADOOP_HOME/etc/hadoop/yarn-site.xml', shell=True)
    nn_ls = NN.findall(NN_LS)[0].split(',')
    rm_ls = RM.findall(RM_LS)[0].split(',')
    for nn in nn_ls:
        cmd = 'hdfs haadmin -getServiceState ' + nn
        print(cmd)
        os.system(cmd)
    for rm in rm_ls:
        cmd = 'yarn rmadmin -getServiceState ' + rm
        print(cmd)
        os.system(cmd)

def start():
    os.system('start-dfs.sh')
    os.system('start-yarn.sh')

def stop():
    os.system('stop-dfs.sh')
    os.system('stop-yarn.sh')

def restart():
    stop()
    time.sleep(9)
    start()

a = os.sys.argv
if len(a) == 2 and a[1] in {'start', 'stop', 'restart'}:
    exec(a[1] + '()')
else:
    state()

ZooKeeper集群启停

touch $ZOOKEEPER_HOME/bin/zookeeper.py
chmod 777 $ZOOKEEPER_HOME/bin/zookeeper.py
vim $ZOOKEEPER_HOME/bin/zookeeper.py

#!/usr/bin/python2
import os, re, subprocess

CMD = 'zkServer.sh '
option = 'status'

a = os.sys.argv
if len(a) == 2:
    a = a[1]
    if a in {'stop', 'start', 'restart'}:
        option = a
cmd = CMD + option

cfg = subprocess.check_output('cat $ZOOKEEPER_HOME/conf/zoo.cfg', shell=True)
for h in re.findall('server.+?=(.+):[0-9]+:[0-9]+', cfg):
    _cmd = 'ssh %s "%s"' % (h, cmd)
    print(_cmd)
    os.system(_cmd)

Kafka集群启停

touch $KAFKA_HOME/bin/kafka.py
chmod 777 $KAFKA_HOME/bin/kafka.py
vim $KAFKA_HOME/bin/kafka.py

#!/usr/bin/python2
import os, re, socket, subprocess, time

def os_system(cmd):
    print(cmd)
    os.system(cmd)

def get_host():
    cfg = subprocess.check_output('cat $KAFKA_HOME/config/server.properties', shell=True)
    zk_ls = re.findall('zookeeper[.]connect=(.+?)/kafka', cfg)[0]
    for hp in zk_ls.split(','):
        h, p = hp.split(':')
        yield h

def stop():
    command = 'kafka-server-stop.sh'
    for h in get_host():
        os_system('ssh %s "%s"' % (h, command))

def start():
    command = 'kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties'
    for h in get_host():
        os_system('ssh %s "%s"' % (h, command))

def restart():
    stop()
    time.sleep(20)
    start()

def status():
    hostname = socket.gethostname()
    cmd = 'kafka-topics.sh --zookeeper {}:2181/kafka --list'.format(hostname)
    os_system(cmd)

a = os.sys.argv
if len(a) == 2:
    a = a[1]
    if a in {'stop', 'start', 'restart', 'status'}:
        exec(a + '()')