kafka+zookeeper+elk自动安装配置

本文介绍了一个自动化部署系统,该系统能够实现数据中心、分析系统和web系统的安装与配置。包括MySQL、Redis、Zookeeper、Kafka等组件的安装,并对配置文件进行修改以适应不同场景的需求。

摘要生成于 C知道 ,由 DeepSeek-R1 满血版支持, 前往体验 >

import os
import string
import re
import subprocess
import time


###############
#获取本地ip
#前提先在/etc/hosts文件配好
###############
import socket
local_ip = socket.gethostbyname(socket.gethostname())
#print(local_ip)

##############
#把data写到文件末尾
##############
def WriteFile(filename, data):
    f =open(filename, "w")
   f.write(data)
   f.close()

##############
#替换单行或全部匹配文件内容
##############
defstring_switch(file_name,replay_line,after_replay_line,replay_line_count=1):
    withopen(file_name, "r") as f:
       #按行读取文件内容
       lines = f.readlines()
    withopen(file_name, "w") as f_w:
       #定义一个数字,用来记录在读取文件时在列表中的位置
       num = 0
       #默认选项,只替换第一次匹配到的行中的字符串
       if replay_line_count == 1:
           for line in lines:
               if replay_line in line:
                   line = line.replace(replay_line,after_replay_line)
                   f_w.write(line)
                   num += 1
                   break
               f_w.write(line)
               num += 1
           #将剩余的文本内容继续输出
           for i in range(num,len(lines)):
               f_w.write(lines[i])
       #全文匹配替换
       elif replay_line_count == 'all':
           for line in lines:
               if chan_line in line:
                   line = line.replace(chan_line,after_replay_line)
               f_w.write(line)


#############
#执行命令输入内容使用
#在本脚本用于pwd当前目录存储
#############
def execCmd(cmd):
    r =os.popen(cmd)
    text =r.read()
    text_strip =text.strip('\n')
   r.close()
    returntext_strip

#############
#更换repo文件内baseurl
#############
baseurl_1 = 'baseurl=file://'
baseurl_2 = execCmd('pwd')
baseurl_3 = '/Package'
baseurl_all = str(baseurl_1+baseurl_2+baseurl_3)
#print (baseurl_all)
repo_file = str(execCmd('pwd')+'/repo_file/gla.repo')
#print(repo_file)
string_switch(repo_file,'baseurl=test',baseurl_all)

#############
#执行命令及等待进程结束在执行下一个进程
#subprocess.STDOUT将错误通过标准输出通道流输出
#为了避免上面进程没有执行完就执行下一个进程
#只有在一个PIPE通道下可以使用communicate等待进程结束在执行下一个进程
#可以使用通道print和use命令输出内容
#父进程与子进程没有频繁交互情况下要这么干
#############
def Command_Sleep(cmd,cmd_id):
    cmd_id =subprocess.Popen(cmd
                             , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
   cmd_id.communicate(input=None)

#############
#repo文件配置及拷贝
#############
Command_Sleep('mkdir /etc/yum.repos.d/bak','cmd_1')
#print(cmd_1.stdout.readlines())
#print(cmd_1.stderr.readlines())
Command_Sleep('mv /etc/yum.repos.d/CentOS-*/etc/yum.repos.d/bak','cmd_2')
Command_Sleep('cp -rf ./repo_file/gla.repo/etc/yum.repos.d/','cmd_3')
Command_Sleep('cp -rf ./repo_file/gla.repo-bak./repo_file/gla.repo','cmd_4')

#############
#提示信息
#############
hint_1 = '\033[1;35;40m请选择需要安装的系统:\033[0m'
hint_2 = '\033[1;35;40m确定没错请按 1~回车,重新输入请按任意字母回车:\033[0m'
hint_3 = '*温馨提示:正在安装请耐心等待'
Error_hint_1 = 'Error:输入的值不存在请重新输入!!!'
Error_hint_2 = 'Error:输入的值格式不正确请重新输入!!!'
Error_hint_3 = '---成功退出---'
Error_hint_4 = '\033[1;35;40mError:您输入的ip格式错误请重新输入!!!\033[0m'
sql_1 = 'mysql -e "grant all on *.* to '+ """'glauser'""" + '@' +"""'localhost'""" + ' identified by' + """ 'gla123'"""+'"'
sql_2 = 'mysql -e "grant all on *.* to '+ """'glauser'""" + '@' +"""'%'""" + ' identified by' + """ 'gla123'"""+'"'
sql_3 = 'mysql -e "create database assets"'
sql_4 = 'mysql -h localhost -u glauser -pgla123 assets < '+execCmd('pwd')+'/sql_file/assets'
print('1.数据中心安装')
print('2.分析系统安装')
print('3.web系统安装')

#############
#zookeeper_install_function
#############
#ip_group可以给许多需要用到三个节点ip的地方
#############
ip_group = {}
es_ip_dic = {}
def zookeeper_install(zookeeper_myid,cmd_id):
    try:
       Command_Sleep('cp -rf ./gla_tar/zookeeper-3.4.8-gla/opt','cmd_18')
       Command_Sleep(zookeeper_myid,cmd_id)
       while True:
           hint_sjzx_ip = input('请输入,数据中心_ip地址:')
           hint_fxxt_ip = input('请输入,分析服务器_ip地址:')
           hint_web_ip = input('请输入,web服务器_ip地址:')
           zookeeper_config_file_dir ='/opt/zookeeper-3.4.8-gla/conf/zoo.cfg'
           if re.match(pattern,hint_sjzx_ip) andre.match(pattern,hint_fxxt_ip) andre.match(pattern,hint_web_ip):
               ip_group['sjzx'] = hint_sjzx_ip
               ip_group['fxxt'] = hint_fxxt_ip
               ip_group['web'] = hint_web_ip
               es_ip_dic[1] = hint_sjzx_ip
               string_switch(zookeeper_config_file_dir,'server.1=',
                             'server.1='+hint_sjzx_ip
                             +':2888:3888')
               string_switch(zookeeper_config_file_dir,'server.2=',
                             'server.1='+hint_fxxt_ip
                             +':2889:3889')
               string_switch(zookeeper_config_file_dir,'server.3=',
                             'server.1='+hint_web_ip
                             +':2890:3890')
               try:
                   while True:
                       out = int(input(hint_2))
                       if out == 1:
                           Command_Sleep('sh /opt/zookeeper-3.4.8-gla/bin/zkServer.sh startzoo.cfg'
                                         ,'cmd_22')
                           break
                       else:
                           print('Eroor:输入的数字参数不存在')
                   break
               except ValueError:
                   Command_Sleep('cp -rf/opt/zookeeper-3.4.8-gla/conf/zoo.cfg-bak'
                                 +'/opt/zookeeper-3.4.8-gla/conf/zoo.cfg','cmd_21')
                   print('\033[1;35;40m请重新输入\033[0m')
           else:
               print(Error_hint_4)
    exceptKeyboardInterrupt:
       print(Error_hint_3)

###############
#kafka&kafkaweb_install_function
###############
def kafka_install(broker_id,replay_broker_id):
   kafka_config_file_dir ='/opt/kafka_2.12-0.10.2.0-gla/config/server.properties'
   Command_Sleep('cp -rf ./gla_tar/kafka_2.12-0.10.2.0-gla/opt','cmd_23')
   string_switch(kafka_config_file_dir,broker_id,replay_broker_id)
   string_switch(kafka_config_file_dir,'zookeeper.connect=:2181,:2181,:2181'
                 ,'zookeeper.connect='+ip_group['sjzx']+':2181,'
                 + ip_group['fxxt']+':2181,'
                 + ip_group['web']+':2181')
   string_switch(kafka_config_file_dir,'host.name='
                 ,'host.name'+local_ip)
    cmd_25 =subprocess.Popen('nohup sh ../bin/kafka-server-start.sh '
                             +kafka_config_file_dir+' &'
                             , shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
   time.sleep(10)
################
#kafkaweb
################
def kafka_web():
   kafka_web_start_dir = '/opt/kafkaweb-gla/start.sh'
   Command_Sleep('cp -rf ./gla_tar/kafkaweb-gla /opt','cmd_24')
   string_switch(kafka_web_start_dir,':2181,:2181,:2181'
                 ,ip_group['sjzx']+':2181,'
                 +ip_group['fxxt']+':2181,'
                 +ip_group['web']+':2181')
    cmd_26 =subprocess.Popen('sh /opt/kafkaweb-gla/start.sh'
                             , shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)

################
#开始安装
################
while True:
    try:
       input_1 = int(input(hint_1))
       if input_1 == 1:
           #############
           #mysql安装和操作
           #############
           Command_Sleep('yum -y install mysql','cmd_28')
           Command_Sleep('yum -y install mysql-server','cmd_46')
           Command_Sleep('/etc/init.d/mysqld start ','cmd_47')
           Command_Sleep(sql_1,'cmd_41')
           Command_Sleep(sql_2,'cmd_42')
           Command_Sleep(sql_3,'cmd_43')
           Command_Sleep(sql_4,'cmd_45')
           #############
           #redis安装
           #############
           Command_Sleep('cp -rf ./gla_tar/redis-el6.art.x86_64-gla'
                         +' /opt','cmd_29')
           Command_Sleep('sh/opt/redis-el6.art.x86_64-gla/install.sh','cmd_30')
           #############
           #zookeeper_install
           #############
           zookeeper_install('echo 1 >/opt/zookeeper-3.4.8-gla/data/myid','cmd_20')
           print(hint_3)
           #############
           #kafka_install
           #############
           kafka_install('broker.id=0','broker.id=0')
           #############
           #es_install
           #############
           Command_Sleep('/etc/init.d/iptables stop','cmd_5')
           Command_Sleep('setenforce 0','cmd_6')
           Command_Sleep('rpm -ivh./gla_tar/jdk-8u121-linux-x64.rpm','cmd_7')
           Command_Sleep('cp -rf /etc/sysctl.conf/etc/sysctl.conf-bak','cmd_8')
           Command_Sleep('cp -rf /etc/security/limits.conf/etc/security/limits.conf-bak','cmd_9')
           Command_Sleep('cp -rf /etc/security/limits.d/90-nproc.conf/etc/security/limits.d/90-nproc.conf-bak'
                         ,'cmd_10')
           Command_Sleep('cp -rf ./other_config_file/sysctl.conf/etc/sysctl.conf','cmd_11')
           Command_Sleep('cp -rf ./other_config_file/limits.conf/etc/security/limits.conf','cmd_12')
           Command_Sleep('cp -rf ./other_config_file/90-nproc.conf/etc/security/limits.d/90-nproc.conf'
                         ,'cmd_13')
           Command_Sleep('useradd glauser','cmd_14')
           Command_Sleep('cp -rf ./gla_tar/elasticsearch-gla-node/opt','cmd_15')
           Command_Sleep('chown -R glauser:glauser/opt/elasticsearch-gla-node','cmd_16')
           cmd_17 = subprocess.Popen('nohup su - glauser -c '
                      +"""'/opt/elasticsearch-gla-node/bin/elasticsearch'""" + '&'
                     , shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
           time.sleep(15)
           ##############
           #fix&kafkadata_jar_app_config
           ##############
           Command_Sleep('cp -rf ./gla_jar/fixassetmonitor//opt','cmd_35')
           Command_Sleep('cp -rf ./gla_jar/kafkaDataPull//opt','cmd_39')
           fix_app_config_file_dir ='/opt/fixassetmonitor/application.properties'
           kafkadata_app_config_file_dir ='/opt/kafkaDataPull/application.properties'
           kafkadata_app_es_file_dir ='/opt/kafkaDataPull/elasticsearch_config.properties'
                #fix
           string_switch(fix_app_config_file_dir,'mysql_ip',ip_group['sjzx'])
           string_switch(fix_app_config_file_dir,'mysql_user','glauser')
           string_switch(fix_app_config_file_dir,'mysql_pass','gla123')
                #kafkadata
           string_switch(kafkadata_app_config_file_dir,'mysql_ip',ip_group['sjzx'])
           string_switch(kafkadata_app_config_file_dir,'mysql_user','glauser')
           string_switch(kafkadata_app_config_file_dir,'mysql_pass','gla123')
           string_switch(kafkadata_app_config_file_dir,'kafka_server1',ip_group['sjzx'])
           string_switch(kafkadata_app_config_file_dir,'kafka_server2',ip_group['fxxt'])
           string_switch(kafkadata_app_config_file_dir,'kafka_server3',ip_group['web'])
           string_switch(kafkadata_app_config_file_dir,'kafka_monitor_ip',ip_group['web'])
           string_switch(kafkadata_app_config_file_dir,'zoo_server1',ip_group['sjzx'])
           string_switch(kafkadata_app_config_file_dir,'zoo_server2',ip_group['fxxt'])
           string_switch(kafkadata_app_config_file_dir,'zoo_server3',ip_group['web'])
           string_switch(kafkadata_app_es_file_dir,'es_server',es_ip_dic[1])
           print('数据中心安装成功')
           break
       elif input_1 == 2:
           ##############
           #zookeeper_install
           ##############
           Command_Sleep('/etc/init.d/iptables stop','cmd_5')
           Command_Sleep('setenforce 0','cmd_6')
           zookeeper_install('echo 2 >/opt/zookeeper-3.4.8-gla/data/myid','cmd_27')
           print(hint_3)
           ##############
           #kafka_install
           ##############
           kafka_install('broker.id=0','broker.id=1')
           ##############
           #logstash安装
           ##############
           Command_Sleep('cp -rf ./gla_tar/logstash-5.2.2-gla'
                         +' /opt','cmd_31')
           logstash_config_file_dir ='/opt/logstash-5.2.2-gla/config/elastic.conf'
           string_switch(logstash_config_file_dir,'127.0.0.1',es_ip_dic[1])
           cmd_32 = subprocess.Popen('nohup/opt/logstash-5.2.2-gla/bin/logstash -f '
                                     +logstash_config_file_dir+' &'
                                     ,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
           #############
           #entty_jar_app_config
           #############
           Command_Sleep('cp -rf ./gla_jar/entty/ /opt','cmd_40')
           entty_app_config_file_dir ='/opt/entty/application.properties'
           string_switch(entty_app_config_file_dir,'kafka_server1',ip_group['sjzx'])
           string_switch(entty_app_config_file_dir,'kafka_server2',ip_group['fxxt'])
           string_switch(entty_app_config_file_dir,'kafka_server3',ip_group['web'])
           string_switch(entty_app_config_file_dir,'zoo_server1',ip_group['sjzx'])
           string_switch(entty_app_config_file_dir,'zoo_server2',ip_group['fxxt'])
           string_switch(entty_app_config_file_dir,'zoo_server3',ip_group['web'])
           print('分析系统安装成功')
           break
       elif input_1 == 3:
           #############
           #zookeeper_install
           #############
           Command_Sleep('/etc/init.d/iptables stop','cmd_5')
           Command_Sleep('setenforce 0','cmd_6')
           zookeeper_install('echo 3 >/opt/zookeeper-3.4.8-gla/data/myid','cmd_28')
           print(hint_3)
           #############
           #kafka_install
           #############
           kafka_install('broker.id=0','broker.id=2')
           #############
           #kafka_web_install
           #############
           kafka_web()
           #############
           #kibana_install
           #############
           Command_Sleep('cp -rf ./gla_tar/kibana-5.2.2-SNAPSHOT-linux-gla/opt','cmd_33')
           kibana_config_file_dir ='/opt/kibana-5.2.2-SNAPSHOT-linux-gla/config/kibana.yml'
           string_switch(kibana_config_file_dir,'127.0.0.1',es_ip_dic[1])
           Command_Sleep('cd/opt/kibana-5.2.2-SNAPSHOT-linux-gla/bin','cmd_35')
           cmd_34 = subprocess.Popen('nohup ./kibana'
                                     + ' &')
           #############
           #ui&nmap_jar_app_config
           #############
           Command_Sleep('cp -rf ./gla_jar/webUI/ /opt','cmd_38')
           webui_app_config_file_dir = '/opt/webUI/application.yml'
           webui_sys_config_file_dir = '/opt/webUI/sysconfig.properties'
           string_switch(webui_app_config_file_dir,'mysql_ip',ip_group['sjzx'])
           string_switch(webui_app_config_file_dir,'mysql_user','glauser')
           string_switch(webui_app_config_file_dir,'mysql_pass','gla123')
           string_switch(webui_app_config_file_dir,'fix_ip',ip_group['sjzx'])
           string_switch(webui_sys_config_file_dir,'es_server',es_ip_dic[1])
           string_switch(webui_sys_config_file_dir,'nmap_ip',ip_group['web'])
           print('web系统安装成功')
           break
       else:
           print(Error_hint_1)
    exceptValueError:
       print(Error_hint_2)
    exceptKeyboardInterrupt:
       print('\n'+Error_hint_3)
       break
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值