手把手教你以二进制方式部署k8s（超详细版本）

liux3528

已于 2025-04-01 17:33:20 修改

阅读量804

点赞数 26

文章标签： kubernetes 容器云原生

于 2025-04-01 16:29:14 首次发布

本文链接：https://blog.youkuaiyun.com/qq_37182070/article/details/146897502

版权

本次以二进制部署k8s高可用集群（1.23）,采用nginx+keepalived部署高可用环境，使用

flannel作为网络插件

一、环境准备

1.准备5台虚拟机

192.168.91.18 master-1
192.168.91.19 master-2
192.168.91.20 master-3
192.168.91.21 node-1
192.168.91.22 node-2

2.修改主机名（所有节点）

#每台分别修改
[root@master-1 ~]# hostnamectl  set-hostname master-1

3.配置hosts（所有节点）

[root@master-1 ~]# cat > /etc/hosts <<EOF
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.91.18  master-1
192.168.91.19  master-2
192.168.91.20  master-3
192.168.91.21  node-1
192.168.91.22  node-2
EOF

4.初始化工具安装（所有节点）

 #配置yum源
[root@master-1 yum.repos.d]# cd /etc/yum.repos.d
[root@master-1 yum.repos.d]# mkdir bakup
[root@master-1 yum.repos.d]# mv *.repo bakup/
[root@master-1 yum.repos.d]# curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
#安装工具
[root@master-1 ~]# yum install net-tools vim wget lrzsz git conntrack-tools -y

5.关闭防火墙与Selinux（所有节点）

[root@master-1 ~]# systemctl stop firewalld
[root@master-1 ~]# systemctl disable firewalld
[root@master-1 ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config

6.设置时区（所有节点）

[root@master-1 ~]# timedatectl set-timezone Asia/Shanghai

7.关闭交换分区（所有节点）

[root@master-1 ~]# swapoff -a
[root@master-1 ~]# sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

8.设置系统时间同步（所有节点）

[root@master-1 ~]# yum install -y chrony
[root@master-1 ~]# systemctl start chronyd
[root@master-1 ~]# systemctl enable chronyd
[root@master-1 ~]# chronyc sources

9.设置免密登录（方便后续拷贝文件）

#本例中从master-1分发
[root@master-1 ~]# yum install -y expect
[root@master-1 ~]# ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa
#密码更换
[root@master-1 ~]# export mypass=12366
[root@master-1 ~]# name=(master-1 master-2 master-3 node-1 node-2)
[root@master-1 ~]# for i in ${name[@]};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$i
  expect {
    \"*yes/no*\" {send \"yes\r\"; exp_continue}
    \"*password*\" {send \"$mypass\r\"; exp_continue}
    \"*Password*\" {send \"$mypass\r\";}
  }"
done

10.优化内核配置（所有节点）

[root@master-1 ~]# cat >>/etc/sysctl.conf<<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
fs.file-max=52706963
fs.nr_open=52706963
EOF
# 默认情况下，如果未指定 kubelet 网络插件，则使用 noop 插件,该插件设置 net/bridge/bridge-nf-call-iptables=1 以确保简单的配置（例如带有网桥的 Docker）与 iptables 代理一起正常工作
[root@master-2 ~]# sysctl -p
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: No such file or directory
net.ipv4.ip_forward = 1
vm.swappiness = 0
fs.file-max = 52706963
fs.nr_open = 52706963

11.配置containerd（node节点）

[root@node-1 ~]# cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
[root@node-1 ~]# modprobe overlay
[root@node-1 ~]# modprobe br_netfilter
[root@node-1 ~]#  sysctl -p
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
fs.file-max = 52706963
fs.nr_open = 52706963

二、前置软件部署

1.编译安装nginx（所有master节点）

1.1 下载、解压nginx

#安装基础软件
[root@master-1 ]# yum -y install pcre pcre-devel openssl openssl-devel gcc gcc-c++ automake autoconf libtool make
#下载
[root@master-1 ]# wget http://nginx.org/download/nginx-1.21.6.tar.gz
#解压
[root@master-1 ~]# tar xf nginx-1.21.6.tar.gz

1.2 编译安装

#配置需要安装的路径以及安装的软件
[root@master-1 ~]# ./configure --prefix=/usr/local/nginx/ \
            --with-pcre \
            --with-http_ssl_module \
            --with-http_stub_status_module \
            --with-stream \
            --with-http_stub_status_module \
            --with-http_gzip_static_module
#编译安装
[root@master-1 nginx-1.21.6]# make -j 4 &&  make install

1.3 使用systemctl管理，并设置开机启动

[root@master-1 ~]# cat >/usr/lib/systemd/system/nginx.service <<'EOF'
[Unit]
Description=The nginx HTTP and reverse proxy server
After=network.target sshd-keygen.service

[Service]
Type=forking
EnvironmentFile=/etc/sysconfig/sshd
ExecStartPre=/usr/local/nginx/sbin/nginx -t -c /usr/local/nginx/conf/nginx.conf
ExecStart=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
ExecReload=/usr/local/nginx/sbin/nginx -s reload
ExecStop=/usr/local/nginx/sbin/nginx -s stop
Restart=on-failure
RestartSec=42s

[Install]
WantedBy=multi-user.target
EOF
[root@master-1 ~]# systemctl enable --now nginx

1.4 修改nginx配置文件

cat > /usr/local/nginx/conf/nginx.conf <<'EOF'
user nginx nginx;
worker_processes auto;
events {
    worker_connections  20240;
    use epoll;
}
error_log /var/log/nginx_error.log info;
stream {
    upstream kube-servers {
        hash $remote_addr consistent;
        server master-1:6443 weight=5 max_fails=1 fail_timeout=3s;
        server master-2:6443 weight=5 max_fails=1 fail_timeout=3s;
        server master-3:6443 weight=5 max_fails=1 fail_timeout=3s;
    }
    server {
        listen 8443 reuseport;
        proxy_connect_timeout 3s;
        proxy_timeout 3000s;
        proxy_pass kube-servers;
    }
}
EOF
[root@master-1 ~]# systemctl reload nginx

1.5 拷贝编译安装好的nginx到其他master节点

#将安装好的nginx拷贝到其他master节点
[root@master-1 local]# cd /usr/local/
[root@master-1 local]# tar -zcvf nginx.tar.gz nginx
[root@master-1 local]# scp nginx.tar.gz master-2:`pwd` 
[root@master-1 local]# scp nginx.tar.gz master-3:`pwd`
[root@master-1 system]# cd /usr/lib/systemd/system
[root@master-1 system]# scp nginx.service master-2:`pwd`

2.部署keepalived（所有master节点）

2.1 下载安装

[root@master-1 ]# yum  -y install  keepalived

2.2 修改配置文件

[root@master-3 ]# cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id 192.168.91.20
}
vrrp_script chk_nginx {
   script "/etc/keepalived/check_nginx.sh"              # 要执行的脚本路径
   interval 1                                           # 脚本执行间隔（秒）
   weight -30                                           # 脚本执行失败时降低的优先级
   fall 2                                               # 连续失败次数阈值
   rise 2                                               # 连续成功次数阈值
   timeout 2                                            # 脚本执行超时时间（秒）
}
vrrp_instance VI_1 {
    state MASTER
    interface ens32
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 192.168.91.20
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.91.254/24 dev ens32
    }
}
EOF

温馨提示:
    router_id:
        节点ip，master每个节点配置自己的IP
    mcast_src_ip:
        节点IP，master每个节点配置自己的IP
    virtual_ipaddress:
        虚拟IP，即VIP。

2.3 编写健康检查脚本 nginx挂之后重启，如重启不了就切VIP

[root@master-3 ]# cat > /etc/keepalived/check_port.sh  <<EOF
A=`ps -C nginx --no-header | wc -l`
if [ $A -eq 0 ];
then
        systemctl restart nginx
        sleep 3
        if [ `ps -C nginx --no-header | wc -l` -eq 0 ]
        then
                systemctl stop keepalived
        fi      
fi 
EOF

#启动keepalived
[root@master-3 ]# systemctl enable --now keepalived

2.4 测试

#测试vip是否正常
[root@master2 ~]# systemctl stop keepalived.service 
[root@master2 ~]# ip a
#手动停止nginx
[root@master2 ~]# systemctl stop nginx
观察ping情况
[root@node-1 ~]# ping 192.168.91.254

3.部署docker（所有node节点）

3.1 软件包下载地址

##软件包下载
[root@node-1 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
#配置yum源
[root@node-1 ~]# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
[root@node-1 ~]# yum install -y docker-ce-24.0.7-1.el7.x86_64 docker-ce-cli-24.0.7-1.el7.x86_64 containerd.io-1.6.26

3.2 启动docker

[root@node-1 ~]# systemctl enable --now docker
[root@node-1 ~]# systemctl  status docker

3.3 配置镜像加速器

[root@node-1 ~]# mkdir -p /etc/docker
[root@node-1 ~]# tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://k0jntw7k.mirror.aliyuncs.com",
                       "https://docker.m.daocloud.io",
                       "https://dockerpull.com",
                       "https://docker.registry.cyou",
                       "https://atomhub.openatom.cn",
                       "https://docker.1panel.live",
                       "https://hub.rat.dev",
                       "https://docker.awsl9527.cn",
                       "https://do.nark.eu.org",
                       "https://docker.ckyl.me",
                       "https://hub.uuuadc.top",
                       "https://docker.chenby.cn"
                      ]
}
EOF

3.4 测试docker是否可以正常拉取镜像

#重启docker
[root@node-1 ~]# systemctl daemon-reload
[root@node-1 ~]# systemctl restart docker
#拉取镜像
[root@node-1 ~]# docker pull nginx

4.部署网络插件flannel （所有节点）

flannel会生成一个配置文件，给每个节点划分网段，docker去读取/run/flannel/subnet.env文件，然后分配容器的ip地址

自动打通每个节点与节点之前的静态路由

docker把数据包丢给flannel-1，flannel-1再发送给另外一个容器

3.1 下载Flannel二进制包

#所有节点,下载到master-1
[root@ master -1 ~]# mkdir /soft && cd /soft
[root@ master -1 ~]# wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
[root@ master -1 ~]# tar xvf flannel-v0.11.0-linux-amd64.tar.gz
[root@ master -1 ~]# mv flanneld mk-docker-opts.sh /usr/local/bin/

#复制flanneld到其他的所有节点
[root@ master -1 ~]# for i in master-2 master-3 node-1 node-2;do scp /usr/local/bin/mk-docker-opts.sh /usr/local/bin/flanneld $i:/usr/local/bin/;done

3.2 配置Flannel

[root@node-1 ~]# mkdir -p /etc/flannel
[root@node-1 ~]# cat > /etc/flannel/flannel.cfg<<EOF
FLANNEL_OPTIONS="-etcd-endpoints=https://192.168.91.18:2379,https://192.168.91.19:2379,https://192.168.91.20:2379 -etcd-cafile=/etc/etcd/ssl/ca.pem -etcd-certfile=/etc/etcd/ssl/server.pem  -etcd-keyfile=/etc/etcd/ssl/server-key.pem"
EOF

3.3 配置flannel 启动文件并启动

cat > /usr/lib/systemd/system/flanneld.service <<EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/etc/flannel/flannel.cfg
ExecStart=/usr/local/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

#启动Flannel
[root@node-2 ~]# systemctl enable --now flanneld
[root@node-2 ~]# systemctl status flanneld

3.4 修改docker 配置文件(所有node节点)

#添加EnvironmentFile=/run/flannel/subnet.env
[root@node-1 ~]# cat >/usr/lib/systemd/system/docker.service<<EOFL
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd  \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target
EOFL

#重启docker
[root@node-1 ~]# systemctl daemon-reload
[root@node-1 ~]# systemctl restart docker

三、k8s各个组件部署

1.etcd安装（所有master节点）

1.1 下载安装自签名证书生成工具

#在分发机器Master-1上操作
[root@master-1 ~]# mkdir /soft && cd /soft
[root@master-1 ~]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
[root@master-1 ~]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
[root@master-1 ~]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
[root@master-1 ~]# chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
[root@master-1 ~]# mv cfssl_linux-amd64 /usr/local/bin/cfssl
[root@master-1 ~]# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
[root@master-1 ~]# mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

1.2 生成etcd证书

#创建目录（Master-1）
[root@master-1 ~]# mkdir /root/etcd && cd /root/etcd
 # CA 证书配置（Master-1）
[root@master-1 ~]# cat << EOF | tee ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

#创建CA证书请求文件（Master-1）
[root@master-1 ~]# cat << EOF | tee ca-csr.json
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF
#创建ETCD证书请求文件
#可以把所有的master IP 加入到csr文件中（Master-1）
[root@master-1 ~]# cat << EOF | tee server-csr.json
{
    "CN": "etcd",
    "hosts": [
    "master-1",
    "master-2",
    "master-3",
    "192.168.91.18",
    "192.168.91.19",
    "192.168.91.20"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

#生成 ETCD CA 证书和ETCD公私钥（Master-1）
[root@master-1 ~]# cd /root/etcd/
#生成ca证书（Master-1）
[root@master-1 etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca –
[root@master-1 etcd]# ll
total 24
-rw-r--r--. 1 root root  287 Mar 17 16:07 ca-config.json
-rw-r--r--. 1 root root  956 Mar 17 16:09 ca.csr
-rw-r--r--. 1 root root  209 Mar 17 16:07 ca-csr.json
-rw-------. 1 root root 1679 Mar 17 16:09 ca-key.pem
-rw-r--r--. 1 root root 1265 Mar 17 16:09 ca.pem
-rw-r--r--. 1 root root  338 Mar 17 16:08 server-csr.json
#生成etcd证书（Master-1）
[root@master-1 etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
[root@master-1 etcd]# ll
total 36
-rw-r--r--. 1 root root  287 Mar 17 16:07 ca-config.json
-rw-r--r--. 1 root root  956 Mar 17 16:09 ca.csr
-rw-r--r--. 1 root root  209 Mar 17 16:07 ca-csr.json
-rw-------. 1 root root 1679 Mar 17 16:09 ca-key.pem
-rw-r--r--. 1 root root 1265 Mar 17 16:09 ca.pem
-rw-r--r--. 1 root root 1054 Mar 17 16:11 server.csr
-rw-r--r--. 1 root root  338 Mar 17 16:08 server-csr.json
-rw-------. 1 root root 1675 Mar 17 16:11 server-key.pem
-rw-r--r--. 1 root root 1379 Mar 17 16:11 server.pem

1.3 部署etcd

[root@master-1 etcd]# cd /soft
[root@master-1 soft]#  wget https://github.com/etcd-io/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz
[root@master-1 soft]# tar -xvf etcd-v3.4.13-linux-amd64.tar.gz
[root@master-1 soft]# cp -p etcd-v3.4.13-linux-amd64/etcd* /usr/local/bin/
#拷贝到其他master机器
[root@master-1 soft]# for i in master-2 master-3;do scp /usr/local/bin/etcd* $i:/usr/local/bin/;done

1.4 编辑etcd配置文件（每台master需单独修改配置）

[root@master-1 soft]# mkdir -p /etc/etcd/cfg/
[root@master-1 soft]# cat >/etc/etcd/cfg/etcd.conf<<EOFL
#[Member]
ETCD_NAME="master-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.91.18:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.91.18:2379,http://192.168.91.18:2390"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.91.18:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.91.18:2379"
ETCD_INITIAL_CLUSTER="master-1=https://192.168.91.18:2380,master-2=https://192.168.91.19:2380,master-3=https://192.168.91.20:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_ENABLE_V2="true"
EOFL
注：
#参数说明:
ETCD_NAME 节点名称, 如果有多个节点, 那么每个节点要修改为本节点的名称。 
ETCD_DATA_DIR 数据目录 
ETCD_LISTEN_PEER_URLS 集群通信监听地址 
ETCD_LISTEN_CLIENT_URLS 客户端访问监听地址 
ETCD_INITIAL_ADVERTISE_PEER_URLS 集群通告地址 
ETCD_ADVERTISE_CLIENT_URLS 客户端通告地址 
ETCD_INITIAL_CLUSTER 集群节点地址,如果多个节点那么逗号分隔
ETCD_INITIAL_CLUSTER="master-1=https://192.168.91.18:2380,master-2=https://192.168.91.19:2380,master-3=https://192.168.91.20:2380"
ETCD_INITIAL_CLUSTER_TOKEN 集群Token 
ETCD_INITIAL_CLUSTER_STATE 加入集群的当前状态，new是新集群，existing表示加入已有集群

1.5 复制etcd证书到指定目录

[root@master-1 soft]# mkdir -p /etc/etcd/ssl/
[root@master-1 soft]# cp /root/etcd/*pem /etc/etcd/ssl/
[root@master-1 soft]# ll /etc/etcd/ssl/
total 16
-rw-------. 1 root root 1679 Mar 17 16:28 ca-key.pem
-rw-r--r--. 1 root root 1265 Mar 17 16:28 ca.pem
-rw-------. 1 root root 1675 Mar 17 16:28 server-key.pem
-rw-r--r--. 1 root root 1379 Mar 17 16:28 server.pem
#复制etcd证书到每个节点(master-1)
[root@master-1 ~]# for i in master-2 master-3 node-1 node-2;do ssh $i mkdir -p /etc/etcd/{cfg,ssl};done
[root@master-1 ~]# for i in master-2 master-3 node-1 node-2;do scp /etc/etcd/ssl/* $i:/etc/etcd/ssl/;done
[root@master-1 ~]# for i in master-2 master-3 node-1 node-2;do echo $i "------>"; ssh $i ls /etc/etcd/ssl;done

1.6 创建ETCD的启动服务

[root@master-1 ~]#  cat > /usr/lib/systemd/system/etcd.service<<EOFL
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/cfg/etcd.conf
ExecStart=/usr/local/bin/etcd \
--initial-cluster-state=new \
--cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--peer-cert-file=/etc/etcd/ssl/server.pem \
--peer-key-file=/etc/etcd/ssl/server-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOFL

1.7 启动etcd

[root@master-1 ~]# systemctl daemon-reload
[root@master-1 ~]# systemctl enable --now etcd
[root@master-1 ~]# systemctl status etcd

1.8 debug模式启动（报错调试）

/usr/local/bin/etcd \
--name=master-1 \
--data-dir=/var/lib/etcd/default.etcd \
--listen-peer-urls=https://192.168.91.18:2380 \
--listen-client-urls=https://192.168.91.18:2379,http://192.168.91.18:2390 \
--advertise-client-urls=https://192.168.91.18:2379 \
--initial-advertise-peer-urls=https://192.168.91.18:2380 \
--initial-cluster=master-1=https://192.168.91.18:2380,master-2=https://192.168.91.19:2380,master-3=https://192.168.91.20:2380 \
--initial-cluster-token=etcd-cluster \
--initial-cluster-state=new \
--cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--peer-cert-file=/etc/etcd/ssl/server.pem \
--peer-key-file=/etc/etcd/ssl/server-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem

1.9 检查etcd集群状态

#方式一
[root@master-2 ~]# etcdctl --cacert=/etc/etcd/ssl/ca.pem \
--cert=/etc/etcd/ssl/server.pem \
--key=/etc/etcd/ssl/server-key.pem \
--endpoints="https://192.168.91.18:2379,https://192.168.91.19:2379,https://192.168.91.20:2379" \
endpoint health 
#方式二
[root@master-2 ~]#  ETCDCTL_API=3 /usr/local/bin/etcdctl \
--write-out=table --cacert=/etc/etcd/ssl/ca.pem \
--cert=/etc/etcd/ssl/server.pem \
--key=/etc/etcd/ssl/server-key.pem \
--endpoints="https://192.168.91.18:2379,https://192.168.91.19:2379,https://192.168.91.20:2379" \
endpoint health
+----------------------------+--------+-------------+-------+
|          ENDPOINT          | HEALTH |    TOOK     | ERROR |
+----------------------------+--------+-------------+-------+
| https://192.168.91.20:2379 |   true | 14.565809ms |       |
| https://192.168.91.18:2379 |   true | 14.713484ms |       |
| https://192.168.91.19:2379 |   true | 14.236667ms |       |
+----------------------------+--------+-------------+-------+

2.0 etcd 设置flannel网段

[root@master-2 ~]#  ETCDCTL_API=2 etcdctl \
--endpoints="https://192.168.91.18:2379,https://192.168.91.19:2379,https://192.168.91.20:2379"  \
--ca-file=/etc/etcd/ssl/ca.pem \
--key-file=/etc/etcd/ssl/server-key.pem  \
--cert-file=/etc/etcd/ssl/server.pem \
set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
#检查是否建立网段
[root@master-2 ~]#  ETCDCTL_API=2 etcdctl \
--endpoints=https://192.168.91.18:2379,https://192.168.91.19:2379,https://192.168.91.20:2379 \
--ca-file=/etc/etcd/ssl/ca.pem \
--cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
get /coreos.com/network/config

#结果
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}

2. k8s证书生成

此证书用于Kubernetes节点之间的通信, 与之前的ETCD证书不同

2.1 创建 Kubernetes 相关证书

[root@master-1 ~]# mkdir /root/kubernetes/ && cd /root/kubernetes/
#配置ca 文件（Master-1）
[root@master-1 ~]# cat << EOF | tee ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

2.2 创建ca证书申请文件（Master-1）

[root@master-1 ~]#  cat << EOF | tee ca-csr.json
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF
#证书字段意义
CN=commonName (网站域名)
OU=organizationUnit (组织部门名)
O=organizationName (组织名)
L=localityName (城市)
S=stateName (省份)
C=country (国家)

2.3 生成API SERVER证书申请文件（Master-1）

#注意要修改VIP的地址
[root@master-1 ~]#  cat << EOF | tee server-csr.json
{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "127.0.0.1",
"10.0.0.2",
"192.168.91.18",
"192.168.91.19",
"192.168.91.20",
"192.168.91.21",
"192.168.91.22",
"192.168.91.254",
"master-1",
"master-2",
"master-3",
"node-1",
"node-2",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

2.4 创建 Kubernetes Proxy 证书申请文件（Master-1）

[root@master-1 ~]#  cat << EOF | tee kube-proxy-csr.json
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "Beijing",
      "ST": "Beijing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

2.5 生成 kubernetes CA 证书和公私钥

# 生成ca证书（Master-1）
[root@master-1 ~]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca –

2.6 生成 api-server 证书（Master-1）

[root@master-1 ~]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
-config=ca-config.json \
-profile=kubernetes server-csr.json | cfssljson -bare server

# cfssl参数
gencert: 生成新的key(密钥)和签名证书
	-initca：初始化一个新ca
	-ca：指明ca的证书
	-ca-key：指明ca的私钥文件
	-config：指明请求证书的json文件
	-profile：与-config中的profile对应，是指根据config中的profile段来生成证书的相关信息

2.7 生成 kube-proxy 证书（Master-1）

[root@master-1 ~]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

2.8 生成的证书列表

[root@node-2 kubernetes]# ll
total 52
-rw-r--r-- 1 root root  294 Apr  1 11:46 ca-config.json
-rw-r--r-- 1 root root 1001 Apr  1 11:57 ca.csr
-rw-r--r-- 1 root root  264 Apr  1 11:46 ca-csr.json
-rw------- 1 root root 1675 Apr  1 11:57 ca-key.pem
-rw-r--r-- 1 root root 1359 Apr  1 11:57 ca.pem
-rw-r--r-- 1 root root 1009 Apr  1 11:58 kube-proxy.csr
-rw-r--r-- 1 root root  230 Apr  1 11:46 kube-proxy-csr.json
-rw------- 1 root root 1679 Apr  1 11:58 kube-proxy-key.pem
-rw-r--r-- 1 root root 1403 Apr  1 11:58 kube-proxy.pem
-rw-r--r-- 1 root root 1358 Apr  1 11:58 server.csr
-rw-r--r-- 1 root root  659 Apr  1 11:46 server-csr.json
-rw------- 1 root root 1679 Apr  1 11:58 server-key.pem
-rw-r--r-- 1 root root 1724 Apr  1 11:58 server.pem

3. 安装kube-apiserver组件（所有master节点）

3.1 下载组件

[root@master-1 soft]# wget https://dl.k8s.io/v1.23.1/kubernetes-server-linux-amd64.tar.gz
[root@master-1 soft]# tar xf kubernetes-server-linux-amd64.tar.gz
[root@master-1 soft]# cd kubernetes/server/bin/
[root@master-1 bin]# cp kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/
#复制执行文件到其他的master节点
[root@master-1 bin]# for i in master-1 master-2 master-3;do scp kube-apiserver kube-controller-manager kube-scheduler kubectl $i:/usr/local/bin/;done
[root@master-1 bin]# for i in node-1 node-2;do scp kubelet kube-proxy $i:/usr/local/bin/;done

3.2 配置Kubernetes证书

#Kubernetes各个组件之间通信需要证书,需要复制个每个master节点（master-1）
[root@master-1  soft]# mkdir -p /etc/kubernetes/{cfg,ssl}
[root@master-1  soft]# cp /root/kubernetes/*.pem /etc/kubernetes/ssl/
#复制到其他的节点(master、node)
[root@master-1  soft]# for i in master-2 master-3 node-1 node-2;do ssh $i mkdir -p /etc/kubernetes/{cfg,ssl};done
[root@master-1  soft]# for i in master-2 master-3 node-1 node-2;do scp /etc/kubernetes/ssl/* $i:/etc/kubernetes/ssl/;done
[root@master-1 bin]# for i in master-2 master-3 node-1 node-2;do echo $i "---------->"; ssh $i ls /etc/kubernetes/ssl;done

3.3 编辑Token 文件(master-1)

#f89a76f197526a0d4bc2bf9c86e871c3:随机字符串,自定义生成; kubelet-bootstrap:用户名; 10001:UID; system:kubelet-bootstrap：用户组
[root@master-1  soft]# vim /etc/kubernetes/cfg/token.csv
f89a76f197526a0d4bc2bf9c86e871c3,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

#复制到其他的master节点(master-1)
[root@master-1 bin]# for i in master-2 master-3;do scp /etc/kubernetes/cfg/token.csv $i:/etc/kubernetes/cfg/token.csv;done

3.4 配置kube-apiserver启动文件

[root@master-1  soft]# cat >/etc/kubernetes/cfg/kube-apiserver.cfg <<EOFL
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount \
--anonymous-auth=false \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=0.0.0.0 \
--insecure-port=0 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.0.0.0/24 \
--token-auth-file=/etc/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/etc/kubernetes/ssl/server.pem  \
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem  \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/server.pem \
--etcd-keyfile=/etc/etcd/ssl/server-key.pem \
--etcd-servers=https://192.168.91.18:2379,https://192.168.91.19:2379,https://192.168.91.20:2379 \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=4"
EOFL

注：
--logtostderr：启用日志
--v：日志等级
--log-dir：日志目录
--etcd-servers：etcd集群地址
--bind-address：监听地址
--secure-port：https安全端口
--advertise-address：集群通告地址
--allow-privileged：启用授权
--service-cluster-ip-range：Service虚拟IP地址段
--enable-admission-plugins：准入控制模块
--authorization-mode：认证授权，启用RBAC授权和节点自管理
--enable-bootstrap-token-auth：启用TLS bootstrap机制
--token-auth-file：bootstrap token文件
--service-node-port-range：Service nodeport类型默认分配端口范围
--kubelet-client-xxx：apiserver访问kubelet客户端证书
--tls-xxx-file：apiserver https证书
--etcd-xxxfile：连接Etcd集群证书
--audit-log-xxx：审计日志

参数说明: https://kubernetes.io/zh/docs/reference/command-line-tools-reference/kube-apiserver/

3.5 配置api-server以systemctl管理启动

[root@master-1 ]# cat >/usr/lib/systemd/system/kube-apiserver.service<<EOFL
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-apiserver.cfg
ExecStart=/usr/local/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOFL
#配置自动启动
[root@master-1 kubernetes]# systemctl enable --now kube-apiserver
[root@master-1 kubernetes]# systemctl status kube-apiserver

3.6 验证

[root@master-1 bin]# curl --insecure https://192.168.91.254:6443
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {
    
  },
  "status": "Failure",
  "message": "Unauthorized",
  "reason": "Unauthorized",
  "code": 401
}

3.7 debug 模式启动（排错）

/usr/local/bin/kube-apiserver \
--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount \
--anonymous-auth=false \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=0.0.0.0 \
--insecure-port=0 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.0.0.0/24 \
--token-auth-file=/etc/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/etc/kubernetes/ssl/server.pem  \
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem  \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/server.pem \
--etcd-keyfile=/etc/etcd/ssl/server-key.pem \
--etcd-servers=https://192.168.91.18:2379,https://192.168.91.19:2379,https://192.168.91.20:2379 \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--feature-gates=RemoveSelfLink=false \
--enable-aggregator-routing=true \
--v=4

4. 部署kubectl（所有master节点）

4.1 创建csr请求文件 (master-1)

[root@master-1 ~]# cd /root/kubernetes
#配置客户端授权文件(master-1)
[root@master-1 ~]# cat >/root/kubernetes/admin-csr.json <<EOFL
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOFL
#说明：
后续 kube-apiserver 使用 RBAC 对客户端(如 kubelet、kube-proxy、Pod)请求进行授权；
kube-apiserver 预定义了一些 RBAC 使用的 RoleBindings，如 cluster-admin 将 Group system:masters 与 Role cluster-admin 绑定，该 Role 授予了调用kube-apiserver 的所有 API的权限；
O指定该证书的 Group 为 system:masters，kubelet 使用该证书访问 kube-apiserver 时 ，由于证书被 CA 签名，所以认证通过，同时由于证书用户组为经过预授权的 system:masters，所以被授予访问所有 API 的权限；
注：这个admin 证书，是将来生成管理员用的kube config 配置文件用的，现在我们一般建议使用RBAC 来对kubernetes 进行角色权限控制， kubernetes 将证书中的CN 字段 作为User， O 字段作为 Group；
"O": "system:masters", 必须是system:masters，否则后面kubectl create clusterrolebinding报错。

4.2 生成客户端证书(master-1)

[root@master-1 kubernetes]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
-config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

#复制证书(master-1)
[root@master-1 kubernetes]# cp /root/kubernetes/admin*.pem /etc/kubernetes/ssl/
[root@master-1 kubernetes]# for i in master-2 master-3;do \
scp /etc/kubernetes/ssl/admin*.pem $i:/etc/kubernetes/ssl/;done

4.3 设置集群参数(master-1)

[root@master-1 kubernetes]# export KUBE_APISERVER="https://192.168.91.254:6443"
[root@master-1 kubernetes]# kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube.config

4.4 设置客户端认证参数(master-1)

[root@master-1 kubernetes]# kubectl config set-credentials admin \
--client-certificate=/etc/kubernetes/ssl/admin.pem \
--client-key=/etc/kubernetes/ssl/admin-key.pem \
--embed-certs=true --kubeconfig=kube.config

4.5 设置上下文参数(master-1)

[root@master-1 kubernetes]# kubectl config set-context kubernetes \
--cluster=kubernetes --user=admin --kubeconfig=kube.config

4.6 设置默认上下文(master-1)

[root@master-1 kubernetes]# kubectl config use-context kubernetes --kubeconfig=kube.config
[root@master-1 kubernetes]# mkdir ~/.kube
[root@master-1 kubernetes]# cp kube.config ~/.kube/config
[root@master-1 kubernetes]# for i in master-2 master-3;do ssh $i "mkdir ~/.kube";done
[root@master-1 kubernetes]# for i in master-2 master-3;do scp ~/.kube/config $i:~/.kube/config;done

4.7 授权kubernetes证书访问kubelet api权限(master-1)

[root@master1 work]# kubectl create clusterrolebinding kube-apiserver:kubelet-apis \
--clusterrole=system:kubelet-api-admin --user kubernetes

4.8 获取集群信息(master-1)

[root@master-1 kubernetes]# kubectl cluster-info
Kubernetes control plane is running at https://192.168.91.254:6443
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.

   #获取组件信息
[root@master-1 bin]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                        ERROR
controller-manager   Unhealthy   Get "https://127.0.0.1:10257/healthz": dial tcp 127.0.0.1:10257: connect: connection refused   
scheduler            Unhealthy   Get "https://127.0.0.1:10259/healthz": dial tcp 127.0.0.1:10259: connect: connection refused   
etcd-0               Healthy     {"health":"true"}                                                                              
etcd-1               Healthy     {"health":"true"}                                                                              
etcd-2               Healthy     {"health":"true"}

5. 部署kube-controller-manager

5.1 创建csr请求文件

[root@master-1 ~]# cd /root/kubernetes
[root@master-1 kubernetes]# cat >/root/kubernetes/kube-controller-manager-csr.json <<EOFL
{
    "CN": "system:kube-controller-manager",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "hosts": [
      "127.0.0.1",
      "192.168.91.18",
      "192.168.91.19",
      "192.168.91.20",
      "192.168.91.254"
    ],
    "names": [
      {
        "C": "CN",
        "ST": "BeiJing",
        "L": "BeiJing",
        "O": "system:kube-controller-manager",
        "OU": "system"
      }
    ]
}
EOFL
注：
hosts 列表包含所有 kube-controller-manager 节点 IP；
CN 为 system:kube-controller-manager、O 为 system:kube-controller-manager，kubernetes 内置的 ClusterRoleBindings system:kube-controller-manager 赋予 kube-controller-manager 工作所需的权限

5.2 生成证书

[root@master-1 kubernetes]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
-config=ca-config.json -profile=kubernetes \
kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager

#查看证书
[root@master-1 kubernetes]# ll kube-controller-manager*.pem
-rw------- 1 root root 1675 Mar 18 10:49 kube-controller-manager-key.pem
-rw-r--r-- 1 root root 1517 Mar 18 10:49 kube-controller-manager.pem

#复制证书
[root@master-1 kubernetes]# cp kube-controller-manager*.pem /etc/kubernetes/ssl/
[root@master-1 kubernetes]# for i in master-2 master-3;do \
scp /etc/kubernetes/ssl/kube-controller-manager*.pem $i:/etc/kubernetes/ssl/;done

5.3 创建kube-controller-manager的kubeconfig

#设置集群参数
[root@master-1 kubernetes]# kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.91.254:6443 \
--kubeconfig=kube-controller-manager.kubeconfig

 #设置客户端认证参数
[root@master-1 kubernetes]# kubectl config set-credentials \
system:kube-controller-manager \
--client-certificate=/etc/kubernetes/ssl/kube-controller-manager.pem \
--client-key=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=kube-controller-manager.kubeconfig

 #设置上下文参数
[root@master-1 kubernetes]# kubectl config set-context \
system:kube-controller-manager --cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=kube-controller-manager.kubeconfig

#设置默认上下文
[root@master-1 kubernetes]# kubectl config use-context \
system:kube-controller-manager \
--kubeconfig=kube-controller-manager.kubeconfig

#复制文件
[root@master-1 kubernetes]# cp kube-controller-manager.kubeconfig /etc/kubernetes/cfg/
[root@master-1 kubernetes]# for i in master-2 master-3;do \
scp /etc/kubernetes/cfg/kube-controller-manager.kubeconfig $i:/etc/kubernetes/cfg;done

5.4 设置配置文件

[root@master-1 bin]# cat >/etc/kubernetes/cfg/kube-controller-manager.cfg<<EOFL
KUBE_CONTROLLER_MANAGER_OPTS="--bind-address=0.0.0.0 \
--secure-port=10257 \
--port=0 \
--address=0.0.0.0 \
--kubeconfig=/etc/kubernetes/cfg/kube-controller-manager.kubeconfig \
--service-cluster-ip-range=10.0.0.0/24 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--allocate-node-cidrs=true \
--cluster-cidr=10.0.0.0/24 \
--experimental-cluster-signing-duration=87600h \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
--leader-elect=true \
--feature-gates=RotateKubeletServerCertificate=true \
--controllers=*,bootstrapsigner,tokencleaner \
--horizontal-pod-autoscaler-sync-period=10s \
--authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics \
--tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
--use-service-account-credentials=true \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2"
EOFL
#注
参数说明: https://kubernetes.io/zh/docs/reference/command-line-tools-reference/kube-controller-manager/

5.4 创建kube-controller-manager 启动文件并启动

[root@master-1 bin]#  cat  >/usr/lib/systemd/system/kube-controller-manager.service<<EOFL
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-controller-manager.cfg
ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOFL
 #启动kube-controller-manager服务
[root@master-1 bin]#  systemctl enable --now kube-controller-manager.service
[root@master-1 bin]#  service kube-controller-manager status
[root@master-1 kubernetes]# journalctl -f -u kube-controller-manager

5.5 debug 启动

/usr/local/bin/kube-controller-manager \
--bind-address=0.0.0.0 \
--secure-port=10257 \
--port=0 \
--address=0.0.0.0 \
--kubeconfig=/etc/kubernetes/cfg/kube-controller-manager.kubeconfig \
--service-cluster-ip-range=10.0.0.0/24 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--allocate-node-cidrs=true \
--cluster-cidr=10.0.0.0/24 \
--experimental-cluster-signing-duration=87600h \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
--leader-elect=true \
--feature-gates=RotateKubeletServerCertificate=true \
--controllers=*,bootstrapsigner,tokencleaner \
--horizontal-pod-autoscaler-sync-period=10s \
--authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics \
--tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
--use-service-account-credentials=true \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2

5.6 查看是否安装成功

[root@master-1 bin]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                        ERROR
scheduler            Unhealthy   Get "https://127.0.0.1:10259/healthz": dial tcp 127.0.0.1:10259: connect: connection refused   
controller-manager   Healthy     ok                                                                                             
etcd-0               Healthy     {"health":"true"}                                                                              
etcd-2               Healthy     {"health":"true"}                                                                              
etcd-1               Healthy     {"health":"true"}

6.部署kube-scheduler

6.1 创建csr请求文件

[root@master-1 kubernetes]# cd /root/kubernetes
[root@master-1 kubernetes]# cat >/root/kubernetes/kube-scheduler-csr.json<<EOFL
{
    "CN": "system:kube-scheduler",
    "hosts": [
      "127.0.0.1",
      "192.168.91.18",
      "192.168.91.19",
      "192.168.91.20",
      "192.168.91.254"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
      {
        "C": "CN",
        "ST": "BeiJing",
        "L": "BeiJing",
        "O": "system:kube-scheduler",
        "OU": "system"
      }
    ]
}
EOFL

注：
hosts 列表包含所有 kube-scheduler 节点 IP；
CN 为 system:kube-scheduler、O 为 system:kube-scheduler，kubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限。

6.2 生成证书

[root@master-1 kubernetes]#  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
-config=ca-config.json -profile=kubernetes \
kube-scheduler-csr.json | cfssljson -bare kube-scheduler
#查看证书
[root@master-1 kubernetes]# ll kube-scheduler*.pem
-rw------- 1 root root 1679 Mar 18 13:01 kube-scheduler-key.pem
-rw-r--r-- 1 root root 1493 Mar 18 13:01 kube-scheduler.pem

#复制证书
[root@master-1 kubernetes]# \cp kube-scheduler*.pem /etc/kubernetes/ssl/
[root@master-1 kubernetes]# for i in master-2 master-3;do \
scp /etc/kubernetes/ssl/kube-scheduler*.pem $i:/etc/kubernetes/ssl/;done

6.3 创建kube-scheduler的kubeconfig

#设置集群参数
[root@master-1 kubernetes]# kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true --server=https://192.168.91.254:6443 \
--kubeconfig=kube-scheduler.kubeconfig

 # 设置客户端认证参数
[root@master-1 kubernetes]# kubectl config set-credentials \
system:kube-scheduler \
--client-certificate=/etc/kubernetes/ssl/kube-scheduler.pem \
--client-key=/etc/kubernetes/ssl/kube-scheduler-key.pem \
--embed-certs=true --kubeconfig=kube-scheduler.kubeconfig

 #设置上下文参数
[root@master-1 kubernetes]# kubectl config \
set-context system:kube-scheduler \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=kube-scheduler.kubeconfig

 #设置默认上下文
[root@master-1 kubernetes]# kubectl config use-context \
system:kube-scheduler \
--kubeconfig=kube-scheduler.kubeconfig

#复制文件
[root@master-1 kubernetes]# \cp kube-scheduler.kubeconfig /etc/kubernetes/cfg/
[root@master-1 kubernetes]# for i in master-2 master-3;do \
scp /etc/kubernetes/cfg/kube-scheduler.kubeconfig $i:/etc/kubernetes/cfg;done

6.4 创建配置文件

[root@master-1 kubernetes]# cat >/etc/kubernetes/cfg/kube-scheduler.cfg<<EOFL
KUBE_SCHEDULER_OPTS="--address=0.0.0.0 \
--kubeconfig=/etc/kubernetes/cfg/kube-scheduler.kubeconfig \
--leader-elect=true \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics \
--v=2"
EOFL

#注
参数说明: https://kubernetes.io/zh/docs/reference/command-line-tools-reference/kube-scheduler/

6.5 创建服务启动文件并启动

root@master-1  soft]# cat >/usr/lib/systemd/system/kube-scheduler.service<<EOFL
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/cfg/kube-scheduler.cfg
ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOFL

启动kube-scheduler服务（所有的master节点）
[root@master-1 kubernetes]# systemctl enable --now kube-scheduler
[root@master-1 kubernetes]# systemctl status kube-scheduler

#查看scheduler状态
[root@master-1 kubernetes]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                  
controller-manager   Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}

7. 部署kubelet（所有node节点）

7.1 创建kubelet-bootstrap.kubeconfig（master1）

[root@master-1 ~]# cd /root/kubernetes
[root@master-1 kubernetes]# BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/cfg/token.csv)

 #设置集群参数
[root@master1 work]# kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true --server=https://192.168.91.254:6443 \
--kubeconfig=kubelet-bootstrap.kubeconfig

 #设置客户端认证参数
[root@master-1 kubernetes]# kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} --kubeconfig=kubelet-bootstrap.kubeconfig

 #设置上下文参数
[root@master-1 kubernetes]# kubectl config set-context default \
--cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig

 #设置默认上下文
[root@master-1 kubernetes]# kubectl config use-context default \
--kubeconfig=kubelet-bootstrap.kubeconfig

 #创建角色绑定
[root@master-1 kubernetes]# kubectl create clusterrolebinding kubelet-bootstrap \
    --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

 #复制文件
[root@master-1 kubernetes]# \cp kubelet-bootstrap.kubeconfig /etc/kubernetes/cfg/
[root@master-1 kubernetes]# for i in node-1 node-2;do \
mkdir /etc/kubernetes/cfg -p;scp /etc/kubernetes/cfg/kubelet-bootstrap.kubeconfig $i:/etc/kubernetes/cfg;done

7.2 设置配置文件(node节点)

#注意修改节点名称
[root@node-1 bin]#  cat > /etc/kubernetes/cfg/kubelet.json<<EOFL
{
  "kind": "KubeletConfiguration",
  "apiVersion": "kubelet.config.k8s.io/v1beta1",
  "authentication": {
    "x509": {
      "clientCAFile": "/etc/kubernetes/ssl/ca.pem"
    },
    "webhook": {
      "enabled": true,
      "cacheTTL": "2m0s"
    },
    "anonymous": {
      "enabled": true
    }
  },
  "authorization": {
    "mode": "Webhook",
    "webhook": {
      "cacheAuthorizedTTL": "5m0s",
      "cacheUnauthorizedTTL": "30s"
    }
  },
  "address": "192.168.91.22",
  "port": 10250,
  "readOnlyPort": 10255,
  "cgroupDriver": "cgroupfs",
  "hairpinMode": "promiscuous-bridge",
  "serializeImagePulls": false,
  "featureGates": {
    "RotateKubeletServerCertificate": true
  },
  "clusterDomain": "cluster.local.",
  "clusterDNS": ["10.0.0.2"]
}
EOFL
注：
–hostname-override：显示名称，集群中唯一
–network-plugin：启用CNI
–kubeconfig：空路径，会自动生成，后面用于连接apiserver
–bootstrap-kubeconfig：首次启动向apiserver申请证书
–config：配置参数文件
–cert-dir：kubelet证书生成目录
–pod-infra-container-image：管理Pod网络容器的镜像

7.3 创建kubelet系统启动文件(node节点)

[root@node-1 bin]# cat >/usr/lib/systemd/system/kubelet.service<<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
  --bootstrap-kubeconfig=/etc/kubernetes/cfg/kubelet-bootstrap.kubeconfig \
  --cert-dir=/etc/kubernetes/ssl \
  --kubeconfig=/etc/kubernetes/cfg/kubelet.config \
  --config=/etc/kubernetes/cfg/kubelet.json \
  --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.2 \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

#注
参数说明: https://kubernetes.io/zh/docs/reference/command-line-tools-reference/kubelet/
[root@node-1 bin]# mkdir /var/lib/kubelet -p
[root@node-1 bin]# mkdir /var/log/kubernetes -p
[root@node-2 bin]# systemctl enable --now kubelet
[root@node-2 bin]# systemctl status kubelet

[root@node-1 bin]# tail -f /var/log/messages  -n 200
Mar 18 15:18:15 node-1 kubelet: I0318 15:18:15.402350   30801 bootstrap.go:150] "No valid private key and/or certificate found, reusing existing private key or creating a new one"
Mar 18 15:18:15 node-1 kubelet: I0318 15:18:15.420810   30801 csr.go:77] csr for this node already exists, reusing
Mar 18 15:18:15 node-1 kubelet: I0318 15:18:15.424474   30801 csr.go:85] csr for this node is still valid
Mar 18 15:18:15 node-1 kubelet: I0318 15:18:15.424503   30801 bootstrap.go:355] "Waiting for client certificate to be issued"

##当看到Waiting for client certificate to be issued时，需要服务器批准请求

7.4 服务端批准与查看CSR请求

#查看CSR请求,Maste-1节点操作
[root@master-1 bin]# kubectl get csr
NAME                                                   AGE   SIGNERNAME                                    REQUESTOR           REQUESTEDDURATION   CONDITION
node-csr-1vSVZBN3dUdk5baIUqozR5mXgmM4pocEvIjmHvnXr_o   39m   kubernetes.io/kube-apiserver-client-kubelet   kubelet-bootstrap   <none>              Pending
node-csr-6PxybvCbJS_VQ94al0G7c7sevkYKV_lMHPBRh6wOG38   43m   kubernetes.io/kube-apiserver-client-kubelet   kubelet-bootstrap   <none>              Pending
#批准请求
#Master节点操作
[root@master-1 ~]# for i in `kubectl get csr | grep Pending | awk '{print $1}'`;do kubectl certificate approve $i;done
[root@master-1 bin]# kubectl get csr
NAME                                                   AGE   SIGNERNAME                                    REQUESTOR           REQUESTEDDURATION   CONDITION
node-csr-1vSVZBN3dUdk5baIUqozR5mXgmM4pocEvIjmHvnXr_o   62m   kubernetes.io/kube-apiserver-client-kubelet   kubelet-bootstrap   <none>              Approved,Issued
node-csr-6PxybvCbJS_VQ94al0G7c7sevkYKV_lMHPBRh6wOG38   66m   kubernetes.io/kube-apiserver-client-kubelet   kubelet-bootstrap   <none>              Approved,Issued

7.5 查看node节点是否部署好

[root@master-1 bin]# kubectl get nodes
NAME     STATUS   ROLES    AGE     VERSION
node-1   Ready    <none>   2m54s   v1.23.1
node-2   Ready    <none>   2m54s   v1.23.1

8.部署kube-proxy（所有node节点）

8.1 创建kubeconfig文件

[root@master-1 ~]# cd /root/kubernetes/
[root@master-1 kubernetes]# kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true --server=https://192.168.91.254:6443 \
--kubeconfig=kube-proxy.kubeconfig

[root@master-1 kubernetes]# kubectl config set-credentials \
kube-proxy --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true --kubeconfig=kube-proxy.kubeconfig

[root@master-1 kubernetes]# kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig

[root@master-1 kubernetes]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

#复制文件
[root@master-1 kubernetes]# \cp kube-proxy.kubeconfig /etc/kubernetes/cfg/
[root@master-1 kubernetes]# for i in node-1 node-2;do \
scp /etc/kubernetes/cfg/kube-proxy.kubeconfig $i:/etc/kubernetes/cfg;done

8.2 创建kube-proxy配置文件

#注意修改节点名称
[root@node-1 ~]# cat >/etc/kubernetes/cfg/kube-proxy.yaml<<EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 192.168.91.22
clientConnection:
  kubeconfig: /etc/kubernetes/cfg/kube-proxy.kubeconfig
clusterCIDR: 10.0.0.0/24
healthzBindAddress: 192.168.91.22:10256
kind: KubeProxyConfiguration
metricsBindAddress: 192.168.91.22:10249
mode: "iptables"
EOF

#注意模式参数，后期修改需要修改此处
mode: "ipvs" 
mode: "iptables"

8.3 创建服务启动文件

[root@node-1 ~]# cat >/usr/lib/systemd/system/kube-proxy.service<<EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
  --config=/etc/kubernetes/cfg/kube-proxy.yaml \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#注 参数说明: https://kubernetes.io/zh/docs/reference/command-line-tools-reference/kube-proxy/
#创建目录
[root@node-1 ~]# mkdir -p /var/lib/kube-proxy
[root@node-1 ~]# mkdir -p /var/log/kubernetes
[root@node-2 bin]# systemctl enable --now kube-proxy 
[root@node-2 bin]# systemctl status kube-proxy

至此，k8s所有组件已安装完成，下面我们来创建一个pod测试一下

四、k8s环境测试

1.创建nginx资源对象

[root@master-2 ~]# kubectl create deployment nginx-demo --image=nginx
deployment.apps/nginx-demo created
[root@master-2 ~]# kubectl get pods
NAME                                      READY   STATUS    RESTARTS        AGE
nginx-demo-5dfc44fcdb-h4bks               1/1     Running   0               8s
[root@master-2 ~]# kubectl scale deployment nginx-demo --replicas 2
deployment.apps/nginx-demo scaled
[root@master-2 ~]# kubectl get pods -o wide
NAME                                      READY   STATUS    RESTARTS        AGE   IP             NODE     NOMINATED NODE   READINESS GATES
nginx-demo-5dfc44fcdb-c6nxb               1/1     Running   0               14s   172.17.54.12   node-1   <none>           <none>
nginx-demo-5dfc44fcdb-h4bks               1/1     Running   0               39s   172.17.42.5    node-2   <none>           <none>

[root@master-2 ~]# curl 172.17.54.12

2. 创建svc资源对象访问

[root@master-2 ~]#  kubectl expose deployment nginx-demo --port=88 --target-port=80 --type=NodePort
service/nginx-demo exposed
[root@master-2 ~]# kubectl get svc
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)        AGE
nginx-demo   NodePort    10.0.0.187   <none>        88:30698/TCP   7s

#访问nginx
http://192.168.91.22:30698/