Docker分布式集群管理

1、安装环境
操作系统：CentOS Linux release 7.2.1511 (Core)

172.16.8.90  
172.16.8.91

2、环境设置

主机名设置
在节点1上执行
echo docker-node1 > /etc/hostname
hostname docker-node1
cat >>/etc/hosts<< EOF
172.16.8.90      docker-node1
172.16.8.91      docker-node2
EOF

在节点2上执行
echo docker-node2 > /etc/hostname
hostname docker-node1
cat >>/etc/hosts<< EOF
172.16.8.90      docker-node1
172.16.8.91      docker-node2
EOF

防火墙&selinux关闭
systemctl stop firewalld.service
systemctl disable firewalld.service
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
grep SELINUX=disabled /etc/selinux/config
setenforce 0

Zookeeper伪集群安装部署
[root@docker-node2 ~]# yum install -y java 

创建软件安装目录
[root@docker-node2 ~]# mkdir /app/tools -p
[root@docker-node2 tools]# wget http://mirrors.cnnic.cn/apache/zookeeper/stable/zookeeper-3.4.9.tar.gz
[root@docker-node2 tools]# tar -xvf zookeeper-3.4.9.tar.gz
[root@docker-node2 tools]# mv zookeeper-3.4.9 /app/
[root@docker-node2 tools]# cd ../
[root@docker-node2 app]# ln -s /app/zookeeper-3.4.9/ /app/zookeeper
[root@docker-node2 app]# cd zookeeper/conf/
[root@docker-node2 conf]# ll
总用量 12
-rw-rw-r-- 1 1001 1001  535 8月  23 15:39 configuration.xsl
-rw-rw-r-- 1 1001 1001 2161 8月  23 15:39 log4j.properties
-rw-rw-r-- 1 1001 1001  922 8月  23 15:39 zoo_sample.cfg
[root@docker-node2 conf]# mv zoo_sample.cfg zoo.cfg
[root@docker-node2 conf]# cat zoo.cfg 
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
dataDir=/tmp/zookeeper
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1


[root@docker-node2 conf]# grep '^[a-z]' zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/app/data/zk1
clientPort=2181
server.1=172.16.8.91:3181:4181
server.2=172.16.8.91:3182:4182
server.3=172.16.8.91:3183:4183

[root@docker-node2 conf]# mkdir /app/data/{zk1,zk2,zk3} -p
[root@docker-node2 conf]# cd /app/data/
[root@docker-node2 data]# ll
总用量 0
drwxr-xr-x 2 root root 6 12月 11 11:27 zk1
drwxr-xr-x 2 root root 6 12月 11 11:27 zk2
drwxr-xr-x 2 root root 6 12月 11 11:27 zk3

[root@docker-node2 data]#  echo "1" > /app/data/zk1/myid
[root@docker-node2 data]#  echo "2" > /app/data/zk2/myid
[root@docker-node2 data]#  echo "3" > /app/data/zk3/myid

[root@docker-node2 app]# cd zookeeper/conf/
[root@docker-node2 conf]# pwd
/app/zookeeper/conf
[root@docker-node2 conf]# cp zoo.cfg zk1.cfg 
[root@docker-node2 conf]# cp zoo.cfg zk2.cfg 
[root@docker-node2 conf]# cp zoo.cfg zk3.cfg 

[root@docker-node2 conf]# sed -i 's#zk1#zk2#g' zk2.cfg
[root@docker-node2 conf]# grep -n dataDir= zk2.cfg
12:dataDir=/app/data/zk2
[root@docker-node2 conf]# sed -i 's#zk1#zk3#g' zk3.cfg
[root@docker-node2 conf]# grep -n dataDir= zk3.cfg
12:dataDir=/app/data/zk3
[root@docker-node2 conf]# sed -i 's#2181#2182#g' zk2.cfg
[root@docker-node2 conf]# grep -n clientPort zk2.cfg
14:clientPort=2182
[root@linux-node2 conf]# sed -i 's#2181#2183#g' zk3.cfg
[root@docker-node2 conf]# grep -n clientPort zk3.cfg
14:clientPort=2183
启动zookeeper
[root@docker-node2 conf]# /app/zookeeper/bin/zkServer.sh start /app/zookeeper/conf/zk1.cfg
ZooKeeper JMX enabled by default
Using config: /app/zookeeper/conf/zk1.cfg
Starting zookeeper ... STARTED
[root@docker-node2 conf]# /app/zookeeper/bin/zkServer.sh start /app/zookeeper/conf/zk2.cfg
ZooKeeper JMX enabled by default
Using config: /app/zookeeper/conf/zk2.cfg
Starting zookeeper ... STARTED
[root@docker-node2 conf]# /app/zookeeper/bin/zkServer.sh start /app/zookeeper/conf/zk3.cfg
ZooKeeper JMX enabled by default
Using config: /app/zookeeper/conf/zk3.cfg
Starting zookeeper ... STARTED

查看当前状态
[root@docker-node2 conf]# /app/zookeeper/bin/zkServer.sh status /app/zookeeper/conf/zk1.cfg
ZooKeeper JMX enabled by default
Using config: /app/zookeeper/conf/zk1.cfg
Mode: follower
[root@docker-node2 conf]# /app/zookeeper/bin/zkServer.sh status /app/zookeeper/conf/zk2.cfg
ZooKeeper JMX enabled by default
Using config: /app/zookeeper/conf/zk2.cfg
Mode: leader
[root@docker-node2 conf]# /app/zookeeper/bin/zkServer.sh status /app/zookeeper/conf/zk3.cfg
ZooKeeper JMX enabled by default
Using config: /app/zookeeper/conf/zk3.cfg
Mode: follower

测试连接Zookeeper
[root@docker-node2 conf]# /app/zookeeper/bin/zkCli.sh -server 172.16.8.91:2181
Connecting to 172.16.8.91:2181
2016-12-11 11:50:34,664 [myid:] - INFO  [main:Environment@100] - Client environment:zookeeper.version=3.4.9-1757313, built on 08/23/2016 06:50 GMT
2016-12-11 11:50:34,666 [myid:] - INFO  [main:Environment@100] - Client environment:host.name=docker-node2
2016-12-11 11:50:34,666 [myid:] - INFO  [main:Environment@100] - Client environment:java.version=1.8.0_111
2016-12-11 11:50:34,668 [myid:] - INFO  [main:Environment@100] - Client environment:java.vendor=Oracle Corporation
2016-12-11 11:50:34,671 [myid:] - INFO  [main:Environment@100] - Client environment:java.home=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.111-1.b15.el7_2.x86_64/jre
2016-12-11 11:50:34,671 [myid:] - INFO  [main:Environment@100] - Client environment:java.class.path=/app/zookeeper/bin/../build/classes:/app/zookeeper/bin/../build/lib/*.jar:/app/zookeeper/bin/../lib/slf4j-log4j12-1.6.1.jar:/app/zookeeper/bin/../lib/slf4j-api-1.6.1.jar:/app/zookeeper/bin/../lib/netty-3.10.5.Final.jar:/app/zookeeper/bin/../lib/log4j-1.2.16.jar:/app/zookeeper/bin/../lib/jline-0.9.94.jar:/app/zookeeper/bin/../zookeeper-3.4.9.jar:/app/zookeeper/bin/../src/java/lib/*.jar:/app/zookeeper/bin/../conf:
2016-12-11 11:50:34,671 [myid:] - INFO  [main:Environment@100] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2016-12-11 11:50:34,671 [myid:] - INFO  [main:Environment@100] - Client environment:java.io.tmpdir=/tmp
2016-12-11 11:50:34,672 [myid:] - INFO  [main:Environment@100] - Client environment:java.compiler=<NA>
2016-12-11 11:50:34,672 [myid:] - INFO  [main:Environment@100] - Client environment:os.name=Linux
2016-12-11 11:50:34,672 [myid:] - INFO  [main:Environment@100] - Client environment:os.arch=amd64
2016-12-11 11:50:34,672 [myid:] - INFO  [main:Environment@100] - Client environment:os.version=3.10.0-327.el7.x86_64
2016-12-11 11:50:34,672 [myid:] - INFO  [main:Environment@100] - Client environment:user.name=root
2016-12-11 11:50:34,672 [myid:] - INFO  [main:Environment@100] - Client environment:user.home=/root
2016-12-11 11:50:34,672 [myid:] - INFO  [main:Environment@100] - Client environment:user.dir=/app/zookeeper-3.4.9/conf
2016-12-11 11:50:34,673 [myid:] - INFO  [main:ZooKeeper@438] - Initiating client connection, connectString=172.16.8.91:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@42110406
Welcome to ZooKeeper!
2016-12-11 11:50:34,709 [myid:] - INFO  [main-SendThread(172.16.8.91:2181):ClientCnxn$SendThread@1032] - Opening socket connection to server 172.16.8.91/172.16.8.91:2181. Will not attempt to authenticate using SASL (unknown error)
JLine support is enabled
2016-12-11 11:50:34,811 [myid:] - INFO  [main-SendThread(172.16.8.91:2181):ClientCnxn$SendThread@876] - Socket connection established to 172.16.8.91/172.16.8.91:2181, initiating session
2016-12-11 11:50:34,846 [myid:] - INFO  [main-SendThread(172.16.8.91:2181):ClientCnxn$SendThread@1299] - Session establishment complete on server 172.16.8.91/172.16.8.91:2181, sessionid = 0x158ebfd18e40000, negotiated timeout = 30000
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
[zk: 172.16.8.91:2181(CONNECTED) 0] 

Mesos_Master部署
[root@docker-node1 ~]# rpm -ivh http://repos.mesosphere.com/el/7/noarch/RPMS/mesosphere-el-repo-7-1.noarch.rpm
[root@docker-node1 ~]# yum -y install mesos marathon 
[root@docker-node1 ~]# cat /etc/mesos/zk 
zk://172.16.8.91:2181,172.16.8.91:2182,172.16.8.91:2183/mesos
[root@docker-node1 ~]# systemctl enable mesos-master mesos-slave marathon
[root@docker-node1 ~]# systemctl start mesos-master mesos-slave marathon
Mesos_Slave部署
[root@docker-node2 conf]#  rpm -ivh http://repos.mesosphere.com/el/7/noarch/RPMS/mesosphere-el-repo-7-1.noarch.rpm
[root@docker-node2 conf]# yum -y install mesos

http://172.16.8.90:5050/

[root@docker-node1 ~]# MASTER=$(mesos-resolve `cat /etc/mesos/zk`)
[root@docker-node1 ~]# mesos-execute --master=$MASTER --name="cluster-test" --command="sleep 60"

[root@docker-node1 ~]# yum -y install docker 


[root@docker-node1 ~]# systemctl enable docker
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.
[root@docker-node1 ~]# systemctl start docker
[root@docker-node1 ~]# docker pull nginx


[root@docker-node1 ~]# echo 'docker,mesos' | tee /etc/mesos-slave/containerizers
docker,mesos
[root@docker-node1 ~]# systemctl restart mesos-slave 


[root@docker-node2 conf]# echo 'docker,mesos' | tee /etc/mesos-slave/containerizers
docker,mesos
[root@docker-node2 conf]# systemctl restart mesos-slave

http://172.16.8.90:8080/

Marathon有自己的REST API，我们通过API的方式来创建一个Nginx的Docker容器。首先创建如下的配置文件nginx.json
[root@docker-node1 ~]# cat nginx.json 
{
"id":"nginx",
"cpus":0.2,
"mem":32.0,
"instances": 1,
"constraints": [["hostname",
"UNIQUE",""]],
"container": {
"type":"DOCKER",
"docker": {
"image": "nginx",
"network": "BRIDGE",
"portMappings": [
{"containerPort": 80,
"hostPort": 0,"servicePort": 0, "protocol":
"tcp" }
]
  }
    }
      }


[root@docker-node1 ~]# curl -X POST http://172.16.8.90:8080/v2/apps -d @/root/nginx.json -H "Content-type:application/json"