Linux HA-OpenStack架构搭建详解

置顶

努力的小T

已于 2024-09-29 16:50:53 修改

阅读量2.4k

点赞数 23

分类专栏： Linux 云计算运维基础文章标签： linux openstack 架构运维服务器云计算

于 2024-09-16 13:27:40 首次发布

本文链接：https://blog.youkuaiyun.com/qq_40797754/article/details/142301870

版权

环境准备

密码统一六个零

主机名	IP
controller1	10.0.0.10
controller2	10.0.0.11
compute1	10.0.0.12
compute2	10.0.0.13
data1	10.0.0.14
data2	10.0.0.15
haproxy1	10.0.0.16
haproxy2	10.0.0.17

客户端系统	虚拟化工具	操作系统
Windows11	VMware15.5pro	Centos7.9

安装基本工具

所有机器

sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config; systemctl disable firewalld

yum install vim iotop bc gcc gcc-c++ glibc glibc-devel pcre pcre-devel openssl  openssl-devel zip unzip zlib-devel  net-tools lrzsz tree ntpdate telnet lsof tcpdump wget libevent libevent-devel bc  systemd-devel bash-completion traceroute bridge-utils  -y

cat >> /etc/hosts << EOF
10.0.0.10 controller1
10.0.0.11 controller2
10.0.0.12 compute1
10.0.0.13 compute2
10.0.0.14 data1
10.0.0.15 data2
10.0.0.16 haproxy1
10.0.0.17 haproxy2

10.0.0.100 openstack.vip.cn
EOF

Haproxy编译部署

haproxy1与haproxy2机器
解决lua环境
- 官网下载：http://www.lua.org/ftp/lua-5.4.4.tar.gz

# 安装基础命令及编译依赖环境
yum install -y gcc readline-devel

mkdir /apps

tar xvf lua-5.4.4.tar.gz -C /apps/

cd /apps/lua-5.4.4/

make linux test

# 查看编译安装的版本
src/lua -v

haproxy1与haproxy2机器
- 官网下载：http://www.haproxy.org/download/2.5/src/haproxy-2.5.7.tar.gz

yum install -y gcc openssl-devel pcre-devel systemd-devel

tar xvf haproxy-2.5.7.tar.gz

cd haproxy-2.5.7/

# 查看安装方法
ll Makefile

cat README

cat INSTALL

# 编译
make -j 4 TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 USE_PCRE=1 USE_SYSTEMD=1 LUA_INC=/apps/lua-5.4.4/src LUA_LIB=/apps/lua-5.4.4/src

# 安装
make install PREFIX=/apps/haproxy

ln -s /apps/haproxy/sbin/haproxy /usr/sbin/

haproxy -v

haproxy启动文件(haproxy1与haproxy2机器)

vim /usr/lib/systemd/system/haproxy.service
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID

[Install]
WantedBy=multi-user.target

mkdir /var/lib/haproxy/

mkdir /etc/haproxy/

# 配置man日志
vim /etc/man_db.conf
MANDATORY_MANPATH     /apps/haproxy/share/man/

# 更新man数据库
mandb

# 配置文件
vim /etc/haproxy/haproxy.cfg
global
   maxconn 100000
   chroot /apps/haproxy
   stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
   #uid 99
   #gid 99
   user haproxy
   group haproxy
   daemon
   # nbproc 4
   # cpu-map 1 0
   # cpu-map 2 1
   # cpu-map 3 2
   # cpu-map 4 3
   pidfile /var/lib/haproxy/haproxy.pid
   log 127.0.0.1 local2 info

defaults
   option http-keep-alive
   option forwardfor
   maxconn 100000
   mode http
   timeout connect 300000ms
   timeout client 300000ms
   timeout server 300000ms

listen stats
   mode http
   bind 0.0.0.0:9999
   stats enable
   log global
   stats uri    /haproxy-status
   stats auth   admin:123456

useradd -r -s /sbin/nologin -d /var/lib/haproxy haproxy


# haproxy需要vip地址才能启动，这里不检测vip也能启动
vim /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1

sysctl -p


systemctl daemon-reload

systemctl enable --now haproxy

Haproxy+Keepalived

安装服务

haproxy1+haproxy2机器

yum install -y keepalived

配置haproxy高可用

haproxy1与haproxy2机器
安装检测工具

yum install  -y psmisc

使用非抢占式
- 效果是当主VIP宕机时VIP飘移过后，重启主VIP也不会将VIP夺回来
haproxy1机器

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
   notification_email {
       360120854@qq.com
       ws3024321237@163.com
   }
   notification_email_from ws3024321237@163.com
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id haprxy1
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_haproxy {
   script "killall -0 haproxy || systemctl restart haproxy"
   interval 2
}

vrrp_instance HA_openstack {
   state BACKUP
   interface eth0
   virtual_router_id 66
   priority 100
   advert_int 2     # 调用脚本两次之间的间隔，默认为1秒
   nopreempt

   track_script {
      check_haproxy
   }

   virtual_ipaddress {
      10.0.0.100/24 dev eth0 label eth0:1       
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}
EOF



vim /etc/keepalived/notify.sh
#!/bin/bash
contact='360120854@qq.com'
notify() {
   
  mailsubject="$(hostname) 切换到 $1, vip 地址发生漂移"    # 发送标题
  mailbody="$(date +'%F %T'): vip发生漂移, $(hostname) 切换到 $1"   # 发送内容
  echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
  notify master
  ;;
backup)
  notify backup
  ;;
fault)
  notify fault
  ;;
*)
  echo "Usage: $(basename $0) {master|backup|fault}"
  exit 1
  ;;
esac

chmod +x /etc/keepalived/notify.sh

yum install -y mailx

vim /etc/mail.rc
''''
set from=360120854@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=360120854@qq.com
set smtp-auth-password=ljroytmuhlkjbgje

systemctl restart keepalived

haproxy2机器

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
   notification_email {
       360120854@qq.com
       ws3024321237@163.com
   }
   notification_email_from ws3024321237@163.com
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id haprxy1
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_haproxy {
   script "killall -0 haproxy || systemctl restart haproxy"
   interval 2
}

vrrp_instance HA_openstack {
   state BACKUP
   interface eth0
   virtual_router_id 66
   priority 80
   advert_int 2
   nopreempt

   track_script {
      check_haproxy
   }

   virtual_ipaddress {
      10.0.0.100/24 dev eth0 label eth0:1       
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}
EOF



vim /etc/keepalived/notify.sh
#!/bin/bash
contact='360120854@qq.com'
notify() {
   
  mailsubject="$(hostname) 切换到 $1, vip 地址发生漂移"    # 发送标题
  mailbody="$(date +'%F %T'): vip发生漂移, $(hostname) 切换到 $1"   # 发送内容
  echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
  notify master
  ;;
backup)
  notify backup
  ;;
fault)
  notify fault
  ;;
*)
  echo "Usage: $(basename $0) {master|backup|fault}"
  exit 1
  ;;
esac

chmod +x /etc/keepalived/notify.sh

yum install -y mailx


vim /etc/mail.rc
''''
set from=360120854@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=360120854@qq.com
set smtp-auth-password=ljroytmuhlkjbgje

systemctl restart keepalived

OpenStack-data

mysql主主架构

采用二进制源码包部署，下载地址：https://downloads.mysql.com/archives/get/p/23/file/mysql-5.7.26-linux-glibc2.12-x86_64.tar.gz
上传至data1机器，部署mysql

yum install -y libaio-devel

yum remove -y mariadb*

tar xf mysql-5.7.26-linux-glibc2.12-x86_64.tar.gz 

useradd -s /sbin/nologin mysql

mkdir /application/mysql -pv

mkdir /data/mysql/data -pv

mkdir /data/mysql/binlog -pv

echo  "PATH=/application/mysql/bin:$PATH" > /etc/profile.d/mysql.sh

source  /etc/profile.d/mysql.sh

mv mysql-5.7.26-linux-glibc2.12-x86_64/* /application/mysql/

mysqld --initialize-insecure  --user=mysql --basedir=/application/mysql --datadir=/data/mysql/data 

vim  /etc/my.cnf 
[mysqld]
user=mysql
basedir=/application/mysql
datadir=/data/mysql/data
socket=/tmp/mysql.sock
server_id=6
port=3306
log_bin=/data/mysql/binlog/mysql-bin
character_set_server=utf8
[mysql]
socket=/tmp/mysql.sock

chown -R mysql.mysql /data/

cp /application/mysql/support-files/mysql.server /etc/init.d/mysqld

service mysqld start

mysql

# 为集群架构准备用户
grant replication slave on *.* to repl@'%' identified by '123';

上传至data2机器，部署mysql

yum install -y libaio-devel

yum remove -y mariadb*

tar xf mysql-5.7.26-linux-glibc2.12-x86_64.tar.gz 

useradd -s /sbin/nologin mysql

mkdir /application/mysql -pv

mkdir /data/mysql/data -pv

mkdir /data/mysql/binlog -pv

echo  "PATH=/application/mysql/bin:$PATH" > /etc/profile.d/mysql.sh

source  /etc/profile.d/mysql.sh

mv mysql-5.7.26-linux-glibc2.12-x86_64/* /application/mysql/

mysqld --initialize-insecure  --user=mysql --basedir=/application/mysql --datadir=/data/mysql/data 

vim  /etc/my.cnf 
[mysqld]
user=mysql
basedir=/application/mysql
datadir=/data/mysql/data
socket=/tmp/mysql.sock
server_id=12
port=3306
log_bin=/data/mysql/binlog/mysql-bin
character_set_server=utf8
[mysql]
socket=/tmp/mysql.sock

chown -R mysql.mysql /data/

cp /application/mysql/support-files/mysql.server /etc/init.d/mysqld

service mysqld start

mysql

# 为集群架构准备用户
grant replication slave on *.* to repl@'%' identified by '123';

主主架构，data2机器执行

mysql

# data1节点查看二进制标识符与位置
show master status;

CHANGE MASTER TO
  MASTER_HOST='data1',
  MASTER_USER='repl',
  MASTER_PASSWORD='123',
  MASTER_PORT=3306,
  MASTER_LOG_FILE='mysql-bin.000001',
  MASTER_LOG_POS=437;
  
# 启动集群
start slave;

# 查看集群状态
show slave status\G

主主架构，data1机器执行

mysql

# data2节点查看二进制标识符与位置
show master status;

CHANGE MASTER TO
  MASTER_HOST='data2',
  MASTER_USER='repl',
  MASTER_PASSWORD='123',
  MASTER_PORT=3306,
  MASTER_LOG_FILE='mysql-bin.000001',
  MASTER_LOG_POS=437;
  
# 启动集群
start slave;

# 查看集群状态
show slave status\G




===========================================
如果出现失败或错误，那么执行如下清理

stop slave;

reset slave;

RabbitMQ集群

data1与data2机器

yum install -y centos-release-openstack-train

yum install -y rabbitmq-server

systemctl enable --now rabbitmq-server

rabbitmqctl add_user openstack 000000

rabbitmqctl set_permissions openstack ".*" ".*" ".*"

rabbitmq-plugins enable rabbitmq_management

data2机器

scp data1:/var/lib/rabbitmq/.erlang.cookie /var/lib/rabbitmq/

systemctl restart rabbitmq-server.service

# 1.停止服务
rabbitmqctl stop_app
# 2.重置状态
rabbitmqctl reset
# 3.节点加入
rabbitmqctl join_cluster rabbit@data1
# 4.启动服务
rabbitmqctl start_app

memcached

data1与data2机器

yum install memcached -y

sed -i "s/127.0.0.1/0.0.0.0/g" /etc/sysconfig/memcached

systemctl enable --now memcached

haproxy配置data机器高可用

data1机器配置如下
- data2暂时不配置，先将data1配置完成复制即可
haproxy代理检测配置详细如下

check #对指定real进行健康状态检查，如果不加此设置，默认不开启检查,check后面没有其它配置也可以启用检查功能
	  #默认对相应的后端服务器IP和端口,利用TCP连接进行周期性健康性检查,注意必须指定端口才能实现健康性检查
 	addr <IP>    #可指定的健康状态监测IP，可以是专门的数据网段，减少业务网络的流量
 	port <num>   #指定的健康状态监测端口
 	inter <num>  #健康状态检查间隔时间，默认2000 ms，单位是毫秒 =2s
 	fall <num>   #后端服务器从线上转为线下的检查的连续失效次数，默认为3
 	rise <num>   #后端服务器从下线恢复上线的检查的连续有效次数，默认为2

haproxy1机器

vim /etc/haproxy/haproxy.cfg
global
   maxconn 100000
   chroot /apps/haproxy
   stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
   #uid 99
   #gid 99
   user haproxy
   group haproxy
   daemon
   # nbproc 4
   # cpu-map 1 0
   # cpu-map 2 1
   # cpu-map 3 2
   # cpu-map 4 3
   pidfile /var/lib/haproxy/haproxy.pid
   log 127.0.0.1 local2 info

defaults
   option http-keep-alive
   option forwardfor
   maxconn 100000
   mode http
   timeout connect 300000ms
   timeout client 300000ms
   timeout server 300000ms

listen stats
   mode http
   bind 0.0.0.0:9999
   stats enable
   log global
   stats uri    /haproxy-status
   stats auth   admin:123456

listen mysql
   bind 10.0.0.100:3306
   mode tcp
   log global
   balance leastconn
   server data1 10.0.0.14:3306 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:3306 check inter 3000 fall 2 rise 5

listen rabbitmq
   bind 10.0.0.100:5672
   mode tcp
   log global
   balance leastconn
   server data1 10.0.0.14:5672 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:5672 check inter 3000 fall 2 rise 5

listen rabbitmq_web
   bind 10.0.0.100:15672
   mode http
   log global
   balance source
   server data1 10.0.0.14:15672 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:15672 check inter 3000 fall 2 rise 5

listen memcached
   bind 10.0.0.100:11211
   mode tcp
   log global
   balance source
   server data1 10.0.0.14:11211 check inter 3000 fall 2 rise 5
   server data2 10.0.0.15:11211 check inter 3000 fall 2 rise 5
   



# 平滑重启haproxy
systemctl reload haproxy.service

chrony部署

所有机器

# 安装时间服务
yum install -y chrony

data1机器

vim /etc/chrony.conf 
server ntp6.aliyun.com iburst
allow all
local stratum 10

systemctl restart chronyd
clock -w

除data1机器所有机器

vim /etc/chrony.conf
server data1 iburst

systemctl restart chronyd
clock -w

keystone部署

data1机器

mysql

CREATE DATABASE keystone;

GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'%' IDENTIFIED BY 'keystone123';

controller1机器

# 安装服务
yum install -y centos-release-openstack-train

yum install -y crudini python-openstackclient openstack-selinux

yum install -y openstack-keystone httpd mod_wsgi python2-PyMySQL python-memcached

# centos8或者其它环境安装区别
python3-PyMySQL
python3-mod_wsgi
#########################


# 备份过滤提前文件
cp /etc/keystone/keystone.conf{
   ,.bak}

grep -Ev "^$|#" /etc/keystone/keystone.conf.bak > /etc/keystone/keystone.conf



# 使用工具配置keystone文件
crudini --set /etc/keystone/keystone.conf database connection mysql+pymysql://keystone:keystone123@openstack.vip.cn/keystone

crudini --set /etc/keystone/keystone.conf token provider fernet



# 同步数据库
su -s /bin/sh -c "keystone-manage db_sync" keystone



# 初始化 Fernet 密钥存储库(提供这些是为了允许在另一个操作系统用户/组下运行 keystone)
keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
keystone-manage credential_setup --keystone-user keystone --keystone-group keystone




# 引导身份
keystone-manage bootstrap --bootstrap-password 000000 \
  --bootstrap-admin-url http://openstack.vip.cn:5000/v3/ \
  --bootstrap-internal-url http://openstack.vip.cn:5000/v3/ \
  --bootstrap-public-url http://openstack.vip.cn:5000/v3/ \
  --bootstrap-region-id RegionOne


# 配置 ServerName选项以引用控制器节点
echo "ServerName controller1" >> /etc/httpd/conf/httpd.conf



# 创建/usr/share/keystone/wsgi-keystone.conf文件的链接
ln -s /usr/share/keystone/wsgi-keystone.conf /etc/httpd/conf.d/


# 开机自启启动服务
systemctl enable --now httpd.service

haproxy1机器配置keystone代理

vim /etc/haproxy/haproxy.cfg
''''''
listen keystone
   bind 10.0.0.100:5000
   mode tcp
   log global
   balance random
   server controller1 10.0.0.10:5000 check inter 3000 fall 2 rise 5
   # controller2高可用没有做，先不要生效，配置完成再取消注释即可
   #server controller2 10.0.0.11:5000 check inter 3000 fall 2 rise 5
   
systemctl reload haproxy.service

controller1机器

cat > /etc/keystone/admin-openrc.sh << EOF
#!/bin/bash
export OS_PROJECT_DOMAIN_NAME=Default
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_NAME=admin
export OS_USERNAME=admin
export OS_PASSWORD=000000
export OS_AUTH_URL=http://openstack.vip.cn:5000/v3
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
EOF



# 创建service 项目
openstack project create --domain default --description "Service Project" service

# 验证
openstack token issue

glance部署

data1机器

mysql

CREATE DATABASE glance;

GRANT ALL PRIVILEGES ON glance.* TO 'glance'@'%' IDENTIFIED BY 'glance123';

controller1机器

# 创建glance用户
openstack user create --domain default --password glance glance

最低0.47元/天解锁文章

Linux HA-OpenStack架构搭建详解

环境准备

密码统一六个零

安装基本工具

所有机器

Haproxy编译部署

haproxy1与haproxy2机器

解决lua环境

haproxy1与haproxy2机器

haproxy启动文件(haproxy1与haproxy2机器)

Haproxy+Keepalived

安装服务

haproxy1+haproxy2机器

配置haproxy高可用

haproxy1与haproxy2机器

安装检测工具

使用非抢占式

haproxy1机器

haproxy2机器

OpenStack-data

mysql主主架构

RabbitMQ集群

memcached

haproxy配置data机器高可用

haproxy1机器

chrony部署

所有机器

data1机器

keystone部署

data1机器

controller1机器

haproxy1机器配置keystone代理

controller1机器

glance部署

data1机器

controller1机器