Redis集群搭建,三台服务器,三主三从。
1. 安装GCC环境
yum install -y gcc-c++
2. 下载redis并解压
cd /usr/local
# 下载Redis
wget http://download.redis.io/releases/redis-5.0.5.tar.gz
# 解压
tar -zxvf redis-5.0.5.tar.gz .
3. 编译Redis源码
#创建节点安装目录
mkdir -p /usr/redis-cluster/7001
cd /usr/local/redis-5.0.5/src
# 编译
make install PREFIX=/usr/redis-cluster/7001
# 拷贝redis.conf文件到节点7001下
cp /usr/local/redis-5.0.5/redis.conf /usr/redis-cluster/7001/bin
4. 修改配置文件
cd /usr/redis-cluster/7001/bin
vim redis.conf
# 关闭仅本机访问
# bind 127.0.0.1
# 端口
port 7001
# 开启集群
cluster-enabled yes
# 关闭保护模式
protected-mode no
# 打开守护进程
daemonize yes
#设置密码
requirepass password
#设置master密码
masterauth password
#持久化
appendonly yes
6. 再复制1个节点,并修改redis.conf端口号为7002
cd /usr/redis-cluster
cp -r 7001 7002
7. 编写启停脚本
先设置三台服务器免密通信,在防火墙规则中放行7001和7002端口。
port.txt
7001
7002
remoteip.txt, server1中为server2和server3的ip,server2中为server1和server3的ip,server2中为server1和server2的ip。
server2的ip
server3的ip
启动本机redis startcurrentserver.sh
#!/bin/sh
ports=`cat /usr/redis-cluster/port.txt`
# 启动本机redis
for port in ${ports}
do
echo "start current server redis at port ${port}"
cd /usr/redis-cluster/${port}/bin
nohup ./redis-server redis.conf >> /usr/redis-cluster/log/${port}.log 2>&1 &
done
启动本机和另两台机器的redis startall.sh
#!/bin/sh
remoteips=`cat /usr/redis-cluster/remoteip.txt`
loginname=root
# 启动本机redis
sh /usr/redis-cluster/startcurrentserver.sh
# 启动远程redis(另一台server)
for remoteip in ${remoteips}
do
echo "开始启动${remoteip} redis..."
ssh ${loginname}@${remoteip} "sh /usr/redis-cluster/startcurrentserver.sh"
echo "启动${remoteip} redis完毕..."
done
停止本机redis stopcurrentserver.sh
#!/bin/sh
ports=`cat /usr/redis-cluster/port.txt`
password=password
# 关闭本机redis
for port in ${ports}
do
echo "start current server redis at port ${port}"
/usr/redis-cluster/${port}/bin/redis-cli -p ${port} -a ${password} shutdown
done
停止本机和另两台机器的redis stopall.sh
#!/bin/sh
remoteips=`cat /usr/redis-cluster/remoteip.txt`
loginname=root
# 启动本机redis
sh /usr/redis-cluster/stopcurrentserver.sh
# 启动远程redis(另一台server)
for remoteip in ${remoteips}
do
echo "开始关闭${remoteip} redis..."
ssh ${loginname}@${remoteip} "sh /usr/redis-cluster/stopcurrentserver.sh"
echo "关闭${remoteip} redis完毕..."
done
添加权限
chmod u+x startcurrentserver.sh
chmod u+x startall.sh
chmod u+x stopcurrentserver.sh
chmod u+x stopall.sh
远程复制给其他server
scp startcurrentserver.sh root@server2:/usr/redis-cluster
scp startcurrentserver.sh root@server3:/usr/redis-cluster
scp startall.sh root@server2:/usr/redis-cluster
scp startall.sh root@server3:/usr/redis-cluster
scp stopcurrentserver.sh root@server2:/usr/redis-cluster
scp stopcurrentserver.sh root@server3:/usr/redis-cluster
scp stopall.sh root@server2:/usr/redis-cluster
scp stopall.sh root@server3:/usr/redis-cluster
8. 启动redis
cd /usr/redis-cluster
sh startall.sh
9. 创建集群
(1) 不指定主从,自动分配
cd /usr/redis-cluster/7001/bin
./redis-cli --cluster create server1:7001 server1:7002 server2:7001 server2:7002 server3:7001 server3:7002 --cluster-replicas 1 -a password
(2) 指定主从
cd /usr/redis-cluster/7001/bin
./redis-cli --cluster create server1:7001 server2:7001 server3:7001 --cluster-replicas 0 -a password
#>>> Performing hash slots allocation on 3 nodes...
#Master[0] -> Slots 0 - 5460
#Master[1] -> Slots 5461 - 10922
#Master[2] -> Slots 10923 - 16383
#M: 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0 server1:7001
# slots:[0-5460] (5461 slots) master
#M: 336c70ac3ab97471da1e2e7cee973cfd456a2131 server2:7001
# slots:[5461-10922] (5462 slots) master
#M: 02f9b70ca33dfa35f0befc8ba2647a9b89682b47 server3:7001
# slots:[10923-16383] (5461 slots) master
#Can I set the above configuration? (type 'yes' to accept): yes
#>>> Nodes configuration updated
#>>> Assign a different config epoch to each node
#>>> Sending CLUSTER MEET messages to join the cluster
#Waiting for the cluster to join
#.
#>>> Performing Cluster Check (using node server1:7001)
#M: 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0 server1:7001
# slots:[0-5460] (5461 slots) master
#M: 02f9b70ca33dfa35f0befc8ba2647a9b89682b47 server3:7001
# slots:[10923-16383] (5461 slots) master
#M: 336c70ac3ab97471da1e2e7cee973cfd456a2131 server2:7001
# slots:[5461-10922] (5462 slots) master
#[OK] All nodes agree about slots configuration.
#>>> Check for open slots...
#>>> Check slots coverage...
#[OK] All 16384 slots covered.
手动添加slave,add-node后第一个节点为要添加的从节点,第二个节点为集群中任意节点,--cluster-master-id为要复制的master的id
cd /usr/redis-cluster/7001/bin
./redis-cli --cluster add-node server2:7002 server1:7001 --cluster-slave --cluster-master-id d0def90314ddc33e8644c0415f6757d5a93d85f5
./redis-cli --cluster add-node server3:7002 server2:7001 --cluster-slave --cluster-master-id 92eb0ae50c2026a18ad085f5ffd5619e40281adb
./redis-cli --cluster add-node server1:7002 server3:7001 --cluster-slave --cluster-master-id 9b837cd0d4f2420166e16d5657f866a3edd67b34
#>>> Adding node server2:7002 to cluster server1:7001
#>>> Performing Cluster Check (using node server1:7001)
#M: 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0 server1:7001
# slots:[0-5460] (5461 slots) master
#M: 02f9b70ca33dfa35f0befc8ba2647a9b89682b47 server3:7001
# slots:[10923-16383] (5461 slots) master
#M: 336c70ac3ab97471da1e2e7cee973cfd456a2131 server2:7001
# slots:[5461-10922] (5462 slots) master
#[OK] All nodes agree about slots configuration.
#>>> Check for open slots...
#>>> Check slots coverage...
#[OK] All 16384 slots covered.
#>>> Send CLUSTER MEET to node server2:7002 to make it join the cluster.
#Waiting for the cluster to join
#
#>>> Configure node as replica of server1:7001.
#[OK] New node added correctly.
#Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
#>>> Adding node server3:7002 to cluster server1:7001
#>>> Performing Cluster Check (using node server1:7001)
#M: 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0 server1:7001
# slots:[0-5460] (5461 slots) master
# 1 additional replica(s)
#M: 02f9b70ca33dfa35f0befc8ba2647a9b89682b47 server3:7001
# slots:[10923-16383] (5461 slots) master
#S: ee4c03259ca4e067dd3a52cc8886ccd116deb9d3 server2:7002
# slots: (0 slots) slave
# replicates 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0
#M: 336c70ac3ab97471da1e2e7cee973cfd456a2131 server2:7001
# slots:[5461-10922] (5462 slots) master
#[OK] All nodes agree about slots configuration.
#>>> Check for open slots...
#>>> Check slots coverage...
#[OK] All 16384 slots covered.
#>>> Send CLUSTER MEET to node server3:7002 to make it join the cluster.
#Waiting for the cluster to join
#
#>>> Configure node as replica of server2:7001.
#[OK] New node added correctly.
#Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
#>>> Adding node server1:7002 to cluster server1:7001
#>>> Performing Cluster Check (using node server1:7001)
#M: 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0 server1:7001
# slots:[0-5460] (5461 slots) master
# 1 additional replica(s)
#M: 336c70ac3ab97471da1e2e7cee973cfd456a2131 server2:7001
# slots:[5461-10922] (5462 slots) master
# 1 additional replica(s)
#S: 49349f5678e0d5d977d299db4e96b44bba480391 server3:7002
# slots: (0 slots) slave
# replicates 336c70ac3ab97471da1e2e7cee973cfd456a2131
#M: 02f9b70ca33dfa35f0befc8ba2647a9b89682b47 server3:7001
# slots:[10923-16383] (5461 slots) master
#S: ee4c03259ca4e067dd3a52cc8886ccd116deb9d3 server2:7002
# slots: (0 slots) slave
# replicates 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0
#[OK] All nodes agree about slots configuration.
#>>> Check for open slots...
#>>> Check slots coverage...
#[OK] All 16384 slots covered.
#>>> Send CLUSTER MEET to node server1:7002 to make it join the cluster.
#Waiting for the cluster to join
#
#>>> Configure node as replica of server3:7001.
#[OK] New node added correctly.
10. 验证集群
cd /usr/redis-cluster/7001/bin
./redis-cli server1 -h -p 7001 -a password -c
server1:7001> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:1456
cluster_stats_messages_pong_sent:1464
cluster_stats_messages_sent:2920
cluster_stats_messages_ping_received:1459
cluster_stats_messages_pong_received:1456
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:2920
server1:7001> cluster nodes
5533c4412f3d43cc28d9d4e5f37ab7335397d0f0 server1:7001@17001 myself,master - 0 1669684777000 1 connected 0-5460
621590c2e5f4858d2f551a87b99d8e57be986ae8 server1:7002@17002 slave 02f9b70ca33dfa35f0befc8ba2647a9b89682b47 0 1669684775000 3 connected
336c70ac3ab97471da1e2e7cee973cfd456a2131 server2:7001@17001 master - 0 1669684777940 2 connected 5461-10922
49349f5678e0d5d977d299db4e96b44bba480391 server3:7002@17002 slave 336c70ac3ab97471da1e2e7cee973cfd456a2131 0 1669684776000 6 connected
02f9b70ca33dfa35f0befc8ba2647a9b89682b47 server3:7001@17001 master - 0 1669684775000 3 connected 10923-16383
ee4c03259ca4e067dd3a52cc8886ccd116deb9d3 server2:7002@17002 slave 5533c4412f3d43cc28d9d4e5f37ab7335397d0f0 0 1669684776936 4 connected
11. 问题总结
在第9步创建完主节点后,出现过两个一样的master id,把700*/bin下的nodes.conf都删除重新创建了,一直显示Waiting for the cluster to join....................................,新建一个连接添加slave,报错。[ERR] Not all 16384 slots are covered by nodes,查看集群信息,也只有一个当前节点,我又各种删除节点重试,整了一个小时没整好,在我合上笔记本准备睡觉的那一刻,我突然想起来了,应该是主节点集群没创建好,我之前在虚拟机上操作就没问题,区别就是现在用的三台云服务器。那一定是服务器间网络或端口不通,网络肯定没问题,都已经可以免密通信了,但是7001和7002都已经放行了,我又看了看nodes.conf里面的信息,有一部分是7001@17001,恍然大悟,redis集群节点之间通信用的是10000+客户端端口。在我去控制台把对应端口也放行后还是不行,因为我刚刚装了个firewalld,用firewalld再执行了下面命令就好了。
firewall-cmd --permanent --add-port=17001/tcp
firewall-cmd --permanent --add-port=17002/tcp
firewall-cmd --reload