lvs+heartbeat实现负载均衡

lvs

• DR模式下：
• 操作环境：

vip :172.25.32.100
server1 172.25.32.1  172.25.32.100
server2 172.25.32.2  172.25.32.100
server3 172.25.32.3  172.25.32.100

server1:

• 安装lvs，指定vip ，以及后端的real server

[root@server1 ~]#  yum search ipvsadm -y
[root@server1 ~]# /etc/init.d/ipvsadm start
[root@server1 ~]# ipvsadm --help
[root@server1 ~]# ipvsadm -A -t 172.25.32.100:80 -s rr   #指定80端口，就访问的是apache
[root@server1 ~]# ipvsadm -a -t 172.25.32.100:80 -r 172.25.32.2 -g
[root@server1 ~]# ipvsadm -a -t 172.25.32.100:80 -r 172.25.32.3 -g
[root@server1 ~]# service ipvsadm save
ipvsadm: Saving IPVS table to /etc/sysconfig/ipvsadm:      [  OK  ]
[root@server1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  172.25.32.100:80 rr
  -> 172.25.32.2:80               Route   1      0          0         
  -> 172.25.32.3:80               Route   1      0          0  
[root@server1 ~]# /etc/init.d/ipvsadm start
[root@server1 ~]#  ip addr add 172.25.32.100 dev eth0
RTNETLINK answers: File exists
[root@server1 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:95:72:9b brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.1/24 brd 172.25.32.255 scope global eth0
    inet 172.25.32.100/32 scope global eth0     #添加的vip 的ip地址
    inet6 fe80::5054:ff:fe95:729b/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 52:54:00:0a:fd:4b brd ff:ff:ff:ff:ff:ff[root@server1 ~]# service ipvsadm reload
ipvsadm: Clearing the current IPVS table:                  [  OK  ]
ipvsadm: Applying IPVS configuration:                      [  OK  ]

server2 和 server3 设定一样

[root@server2 ~]# service httpd start
Starting httpd: 
[root@server2 ~]# ip addr add 172.25.32.100 dev eth0
[root@server2 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:09:b8:85 brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.2/24 brd 172.25.32.255 scope global eth0
    inet 172.25.254.66/16 brd 172.25.255.255 scope global eth0
    inet 172.25.32.100/32 scope global eth0
    inet6 fe80::5054:ff:fe09:b885/64 scope link 
       valid_lft forever preferred_lft forever
[root@server2 ~]# service httpd status
httpd (pid  1529) is running...
[root@server2 ~]# yum install arptables_jf -y
[root@server2 ~]#  arptables -A IN -d 172.25.32.100 -j DROP
[root@server2 ~]# arptables -A OUT -s 172.25.32.100 -j  mangle --mangle-ip-s 172.25.32.2
[root@server2 ~]# /etc/init.d/arptables_jf save
Saving current rules to /etc/sysconfig/arptables:          [  OK  ]
[root@server2 ~]# cat /etc/sysconfig/arptables
# Generated by arptables-save v0.0.8 on Tue Sep 19 11:56:17 2017
*filter
:IN ACCEPT [7021:196588]
:OUT ACCEPT [194:5432]
:FORWARD ACCEPT [0:0]
[1:28] -A IN -d 172.25.32.100 -j DROP 
[0:0] -A OUT -s 172.25.32.100 -j mangle --mangle-ip-s 172.25.32.2 
COMMIT
# Completed on Tue Sep 19 11:56:17 2017

server3:

• 配置和server2一样

• 测试：

[root@foundation32 Desktop]#  for i in {1..15}; do curl 172.25.32.100 ;done
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
[root@foundation32 Desktop]#  for i in {1..15}; do curl 172.25.32.100 ;done
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
[root@foundation32 Desktop]#  for i in {1..15}; do curl 172.25.32.100 ;done
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>

lvs 对真实服务器健康检查：

server1:

[root@server1 ~]# cd /
[root@server1 /]# ls
bbs   dev   ldirectord-3.9.5-3.1.x86_64.rpm  lost+found  opt   sbin     sys  var
bin   etc   lib                              media       proc  selinux  tmp  www
boot  home  lib64                            mnt         root  srv      usr
[root@server1 /]# yum intall ldirectord-3.9.5-3.1.x86_64.rpm -y
[root@server1 /]# rpm -ql ldirectord
/etc/ha.d
/etc/ha.d/resource.d
/etc/ha.d/resource.d/ldirectord
/etc/init.d/ldirectord
/etc/logrotate.d/ldirectord
/usr/lib/ocf/resource.d/heartbeat/ldirectord
/usr/sbin/ldirectord
/usr/share/doc/ldirectord-3.9.5
/usr/share/doc/ldirectord-3.9.5/COPYING
/usr/share/doc/ldirectord-3.9.5/ldirectord.cf
/usr/share/man/man8/ldirectord.8.gz
[root@server1 /]# cp /usr/share/doc/ldirectord-3.9.5/ldirectord.cf /etc/ha.d/
[root@server1 /]# vim /etc/ha.d/ldirectord.cf 
 24 # Sample for an http virtual service
 25 virtual=172.25.32.100:80
 26         real=172.25.32.2:80 gate
 27         real=172.25.32.3:80 gate  
 28         fallback=127.0.0.1:80 gate  #备用访问
 29         service=http
 30         scheduler=rr
 31         #persistent=600
 32         #netmask=255.255.255.255
 33         protocol=tcp
 34         checktype=negotiate
 35         checkport=80
 36         request="index.html"
 37 #       receive="Test Page"
 38         virtualhost=www.x.y.z
 39 
[root@server1 /]# /etc/init.d/ldirectord restart
Restarting ldirectord... success

server3

[root@server3 ~]# service httpd start
Starting httpd: 
[root@server3 ~]# service httpd status
httpd (pid  1512) is running...

server2

[root@server2 ~]# service httpd start
Starting httpd: httpd: Could not reliably determine the server's fully qualified domain name, using 172.25.32.2 for ServerName
                                                           [  OK  ]
[root@server2 ~]# service httpd status
httpd (pid  5119) is running...

• 理论上http请求过来会在server2和server3上轮询访问，server1作为备用，只有在server2和server3都挂掉了才会接替工作。

• 测试：

• 关闭server2上的httpd服务，理论上只会访问server3

[root@server3 ~]# service httpd stop
Stopping httpd:                                            [  OK  ]
[root@foundation32 Desktop]#  for i in {1..15}; do curl 172.25.32.100 ;done
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>
<h2>www.linux.org apache-server3</h2>

• 关闭server3上的httpd服务，理论上只会访问server2

[root@server3 ~]# service httpd stop
Stopping httpd:                                            [  OK  ]

[root@server2 ~]# service httpd start
Starting httpd: httpd: Could not reliably determine the server's fully qualified domain name, using 172.25.32.2 for ServerName
                                                           [  OK  

[root@foundation32 Desktop]#  for i in {1..15}; do curl 172.25.32.100 ;done
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>
<h1>www.linux.org-apache-server2</h1>

• server2和server3的httpd都关掉，理论上只访问备份服务机,即server1（server1的httpd服务要开启）

[root@server2 ~]# service httpd stop
Stopping httpd:                                            [  OK  ]
[root@server2 ~]# 
[root@server3 ~]# service httpd stop
Stopping httpd:                                            [  OK  ]
[root@server3 ~]# 
[root@foundation32 Desktop]#  for i in {1..15}; do curl 172.25.32.100 ;done
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>
<h1>www.linux.org-server1</h1>

• server2和server3 都开启，两个会轮询访问，而server1不会被访问：

[root@server3 ~]# service httpd start
Starting httpd: httpd: Could not reliably determine the server's fully qualified domain name, using 172.25.32.3 for ServerName
                                                           [  OK  ]

[root@server2 ~]# service httpd start
Starting httpd: httpd: Could not reliably determine the server's fully qualified domain name, using 172.25.32.2 for ServerName
                                                           [  OK  ]
[root@foundation32 Desktop]#  for i in {1..15}; do curl 172.25.32.100 ;done
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>
<h2>www.linux.org apache-server3</h2>
<h1>www.linux.org-apache-server2</h1>

heartbeat+lvs

server1和server4：安装ldirectord的两台服务机,没有vip地址
server2和server3：两台apache服务器，有vip地址172.25.32.100

配置

server1和server4的配置相同

[root@server1 rpms]# ls
heartbeat-3.0.4-2.el6.x86_64.rpm  heartbeat-libs-3.0.4-2.el6.x86_64.rpm  ldirectord-3.9.5-3.1.x86_64.rpm
[root@server1 rpms]# yum install heartbeat* ldirectord-3.9.5-3.1.x86_64.rpm -y
[root@server1 rpms]# rpm -ql heartbeat
/etc/ha.d
/etc/ha.d/README.config
/etc/ha.d/harc
/etc/ha.d/rc.d
...
/usr/share/doc/heartbeat-3.0.4/authkeys
/usr/share/doc/heartbeat-3.0.4/ha.cf
/usr/share/doc/heartbeat-3.0.4/haresources
...
[root@server1 rpms]# cp  /usr/share/doc/heartbeat-3.0.4/authkeys /etc/ha.d/
[root@server1 rpms]# cp /usr/share/doc/heartbeat-3.0.4/haresources /etc/ha.d/
[root@server1 rpms]# cp /usr/share/doc/heartbeat-3.0.4/ha.cf /etc/ha.d/
[root@server1 ha.d]# cat /etc/ha.d/haresources | grep -v "#" | grep -v ";" | grep -v "^$" #编辑成如下
server1 IPaddr::172.25.32.100/24/eth0   httpd ldirectord 
[root@server1 ha.d]# cat /etc/ha.d/authkeys | grep -v "#" | grep -v ";" | grep -v "^$" #编辑成如下
auth 1
1 crc
[root@server1 ha.d]# cat /etc/ha.d/ha.cf | grep -v "#" | grep -v ";" | grep -v "^$" #编辑成如下
logfacility local0
keepalive 1
deadtime 30
warntime 10
initdead 60
udpport 694
auto_failback on
node    server1
node    server4
ping 172.25.32.250
respawn hacluster /usr/lib64/heartbeat/ipfail
apiauth ipfail gid=haclient uid=hacluster
[root@server1 ha.d]# chmod 600 /etc/ha.d/authkeys 

• 集群中要完成的服务要让集群来启动，所以原来手动开启的现在关闭，两端（1和4）都关闭

[root@server1 ha.d]# /etc/init.d/ldirectord stop
Stopping ldirectord... success
[root@server1 ha.d]# /etc/init.d/httpd stop
Stopping httpd:                                            [FAILED]
[root@server1 ha.d]# /etc/init.d/httpd status
httpd is stopped

• 在server4上进行同样的配置

• 理论上server1没有心跳（heartbeat故障），就会自动跳转到server4上

• 测试：

[root@server4 ha.d]#  ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:34:98:fb brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.4/24 brd 172.25.32.255 scope global eth0   #vip不在server4上面，肯定在server1上
    inet6 fe80::5054:ff:fe34:98fb/64 scope link 
       valid_lft forever preferred_lft forever
[root@server1 ha.d]#  ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:95:72:9b brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.1/24 brd 172.25.32.255 scope global eth0
    inet 172.25.32.100/24 brd 172.25.32.255 scope global secondary eth0  #vip地址，本身没有这个ip
    inet6 fe80::5054:ff:fe95:729b/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 52:54:00:0a:fd:4b brd ff:ff:ff:ff:ff:ff

• 现在在server1上停止掉heartbeat服务，理论上vip会跳到server4上

[root@server1 ha.d]# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.

[root@server1 ha.d]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:95:72:9b brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.1/24 brd 172.25.32.255 scope global eth0
    inet6 fe80::5054:ff:fe95:729b/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link
[root@server4 ha.d]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:34:98:fb brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.4/24 brd 172.25.32.255 scope global eth0
    inet 172.25.32.100/24 brd 172.25.32.255 scope global secondary eth0
    inet6 fe80::5054:ff:fe34:98fb/64 scope link 
       valid_lft forever preferred_lft forever
/ether 52:54:00:0a:fd:4b brd ff:ff:ff:ff:ff:ff

• 现在在server4上停止掉heartbeat服务，理论上vip会跳到server1上(前提是心跳服务开启)

[root@server4 ha.d]# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.
[root@server4 ha.d]#  ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:34:98:fb brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.4/24 brd 172.25.32.255 scope global eth0
    inet6 fe80::5054:ff:fe34:98fb/64 scope link 
       valid_lft forever preferred_lft forever
[root@server1 ha.d]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 52:54:00:95:72:9b brd ff:ff:ff:ff:ff:ff
    inet 172.25.32.1/24 brd 172.25.32.255 scope global eth0
    inet 172.25.32.100/24 brd 172.25.32.255 scope global secondary eth0
    inet6 fe80::5054:ff:fe95:729b/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 52:54:00:0a:fd:4b brd ff:ff:ff:ff:ff:ff