基于docker swarm集群实验 -- 2020/8/10

最新推荐文章于 2024-07-19 09:12:44 发布

原创最新推荐文章于 2024-07-19 09:12:44 发布 · 591 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#docker

linux 同时被 3 个专栏收录

18 篇文章

订阅专栏

docker

5 篇文章

订阅专栏

nginx

3 篇文章

订阅专栏

本文介绍了如何搭建Docker Swarm集群，包括在4台机器上安装Docker、配置Swarm、制作Nginx镜像、创建NFS数据卷以及设置负载均衡。通过两台负载均衡器（keepalived+nginx）实现对两台realserver的7层负载均衡，详细阐述了配置过程和验证方法。

docker集群

1、安装好4台docker机器（1c2g）

docker-maneger:192.168.80.1
docker-2:192.168.80.2
docker-3:192.168.80.3
docker-4:192.168.80.4
LB-1:ens33:192.168.0.207  ens37:192.168.80.208

2、开启路由功能

$ vim snat.sh

iptables -F
iptables -t nat -F
echo 1 >/proc/sys/net/ipv4/ip_forward
iptables -t nat -A POSTROUTING -s 192.168.80.0/24 -o ens33 -j SNAT --to-source 192.168.0.207

[root@LB-1 ~]# bash snat.sh 
[root@LB-1 ~]# iptables -L -t nat -vn
Chain PREROUTING (policy ACCEPT 1 packets, 147 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain INPUT (policy ACCEPT 1 packets, 147 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain OUTPUT (policy ACCEPT 1 packets, 108 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain POSTROUTING (policy ACCEPT 1 packets, 108 bytes)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 SNAT       all  --  *      ens33   192.168.80.0/24      0.0.0.0/0            to:192.168.0.207

关闭防火墙、selinux

systemctl stop firewalld
systemctl disable firewalld
sed -i's/=enforcing/=disabled/'

3、在管理节点docker机器上配置swarm

docker swarm init --advertise-addr 192.168.80.1

在工作节点上加入

    docker swarm join \
    --token SWMTKN-1-4crsq2chuxhtd4a1bb0eo0bh1gpvgjchgx6141zhe7xe0oh8of-0nwyw4o5l3rimiv6c3jsd2yl4 \
    192.168.80.1:2377

4、获取centos7的镜像

docker pull docker.io/sglim2/centos7

5、在manager节点上配置Dockerfile，自制nginx镜像

FROM docker.io/sglim2/centos7

MAINTAINER Ekko

ENV company dtstack
WORKDIR /
COPY nginx-1.18.0.tar.gz /

RUN yum install -y zlib zlib-devel openssl-devel pcre pcre-devel gcc gcc-c++ autoconf automake make \
    && tar xf nginx-1.18.0.tar.gz \
    && cd nginx-1.18.0 \
    && ./configure  --prefix=/usr/local/nginx --with-threads  --with-file-aio  --with-http_ssl_module  --with-http_stub_status_module  --with-stream \
    && make -j 2 ; make install \
    && sed -i '44 c root /app ;' /usr/local/nginx/conf/nginx.conf
    
ENV PATH /usr/local/nginx/sbin:$PATH

EXPOSE 80

STOPSIGNAL SIGTERM

ENTRYPOINT ["nginx"]

CMD ["-g","daemon off;"]

开始自制

docker build -t ekko_nginx .

使用自制的镜像启动容器

[root@docker01-manager nginx]# docker images
REPOSITORY                 TAG                 IMAGE ID            CREATED             SIZE
ekko_nginx                 latest              b0a0e9f5a393        36 seconds ago      812 MB
docker.io/nginx            latest              8cf1bfb43ff5        4 days ago          132 MB
docker.io/sglim2/centos7   latest              63bd2b510f17        3 years ago         543 MB
[root@docker01-manager nginx]# docker run -d -p 8080:80 --name nginx1 ekko_nginx
d36b3c08eec6005f53847a69fb85166a97daa0207d1f0e3b7d6c91bc7724ef4c
[root@docker01-manager nginx]# docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                  NAMES
d36b3c08eec6        ekko_nginx          "nginx -g 'daemon ..."   5 seconds ago       Up 4 seconds        0.0.0.0:8080->80/tcp   nginx

可以进入容器在/app下创建首页

[root@docker01-manager nginx]# docker exec -it nginx1 /bin/bash
[root@d36b3c08eec6 /]# ls
anaconda-post.log  dev  home  lib64       media  nginx-1.18.0         opt   root  sbin  sys  usr
bin                etc  lib   lost+found  mnt    nginx-1.18.0.tar.gz  proc  run   srv   tmp  var
[root@d36b3c08eec6 /]# mkdir -p app 
[root@d36b3c08eec6 /]# ls
anaconda-post.log  bin  etc   lib    lost+found  mnt           nginx-1.18.0.tar.gz  proc  run   srv  tmp  var
app                dev  home  lib64  media       nginx-1.18.0  opt                  root  sbin  sys  usr
[root@d36b3c08eec6 /]# cd app
[root@d36b3c08eec6 app]# ls
[root@d36b3c08eec6 app]# echo ekko hello >index.html
[root@d36b3c08eec6 app]# ls
index.html

[root@docker01-manager nginx]# curl 192.168.80.1:8080
ekko hello

配置好nfs，在nfs机器上：

yum install nfs-utils -y
systemctl stop firewalld
systemctl disable firewalld

[root@nfs ~]# systemctl start nfs-server
[root@nfs ~]# systemctl enable nfs-server
Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.

配置export

[root@nfs ~]# vi /etc/exports
[root@nfs ~]# cat /etc/exports
/web 192.168.80.*(rw,all_squash,async)

[root@nfs web]# exportfs -rv
exporting 192.168.80.*:/web 

[root@nfs ~]# chown nfsnobody:nfsnobody /web/
[root@nfs ~]# ll -d /web/
drwxr-xr-x. 2 nfsnobody nfsnobody 24 7月  26 21:19 /web/

在docker-manager上挂在nfs

yum install -y nfs-server
mount 192.168.80.5:/web /web

vim /etc/fstab
192.168.80.5:/web  /web  nfs  defaults 0 0

创建nfs数据卷

[root@docker01-manager web]# docker volume create --driver local  --opt type=nfs --opt o=addr=192.168.80.5,rw --opt device=:/web sc
sc
[root@docker01-manager web]# docker volume ls
DRIVER              VOLUME NAME
local               sc

启动容器使用sc数据卷

docker run -d -p 80:80 --name sc_nginx -v sc:/app ekko_nginx

在work节点机器上把ekko_nginx镜像导入

在manager节点上创建服务, 起5个副本

docker service create --name nfs-sc-nginx --publish 80:80 --mount type=volume,source=sc,destination=/app --replicas 5 ekko_nginx

扩大副本数量 – 动态扩容运用于双十一时期

docker service scale nfs-sc-nginx=10

nginx做负载均衡

**两台负载均衡器(keepalived+nginx)😗*192.168.31.103 192.168.31.251

**两台realserver:**192.168.31.22 192.168.31.244

systemctl stop firewalld 
systemctl disable firewalld
setenforce 0

把realserver的ip配置到负载均衡上：这是7层负载均衡配置

        
http {
    upstream tzk_cluster {
        server 192.168.31.22;
        server 192.168.31.244;
    }
    
    server {
    	...
        location / {
            proxy_pass http://tzk_cluster;
        }
    }
}

访问查看轮询效果：

curl 192.168.31.103
curl 192.168.31.251

yum install keepalived
systemctl enable keepalived

vim /etc/keepalived/keepalived.conf

192.168.31.103

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.88
    }
}

[root@nginx-keepalived-lb1 keepalived]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:73:bb:af brd ff:ff:ff:ff:ff:ff
    inet 192.168.31.103/24 brd 192.168.31.255 scope global noprefixroute dynamic ens33
       valid_lft 32647sec preferred_lft 32647sec
    inet 192.168.31.88/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::a52d:da1a:fc97:7524/64 scope link noprefixroute
       valid_lft forever preferred_lft forever

192.168.31.251

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.88
    }
}

[root@nginx-keepalived-lb2 keepalived]# ip ad
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:f3:32:d0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.31.251/24 brd 192.168.31.255 scope global noprefixroute dynamic ens33
       valid_lft 34121sec preferred_lft 34121sec
    inet6 fe80::e7c8:b572:63ba:479/64 scope link noprefixroute
       valid_lft forever preferred_lft forever

可systemctl stop keepalived, 查看192.168.31.251的ip add – 飘移

[root@nginx-keepalived-lb2 keepalived]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:f3:32:d0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.31.251/24 brd 192.168.31.255 scope global noprefixroute dynamic ens33
       valid_lft 33968sec preferred_lft 33968sec
    inet 192.168.31.88/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::e7c8:b572:63ba:479/64 scope link noprefixroute
       valid_lft forever preferred_lft forever