k8s 1.18.2 kubeadm部署单master_kuberneters 1.18.2编译后没有kubeadm-优快云博客

本文链接：https://blog.youkuaiyun.com/qq_30950597/article/details/132385747

K8s 1.18.2 kubeadm安装

1·初始化系统：

# 关闭firewalld 、dnsmasq、selinux(CentOS7需要关闭NetworkManager，CentOS8不需要) 
# 关闭防火墙 优化系统内核
systemctl stop firewalld && systemctl disable --now firewalld
systemctl disable --now dnsmasq 
setenforce 0  
sed -i "s/^SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config
# sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
# 关闭swap分区，fstab注释swap 
swapoff -a && sysctl -w vm.swappiness=0  
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# sed -ri 's/.*swap.*/#&/' /etc/fstab
# 系统优化
ulimit -SHn 65535
cat >> /etc/security/limits.conf <<- EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* seft memlock unlimited
* hard memlock unlimitedd
EOF
# 暗转基本组件
yum install wget jq psmisc vim net-tools yum-utils device-mapper-persistent-data lvm2 git -y
# CentOS 7安装yum源如下
cat >>  /etc/resolv.conf <<- EOF
nameserver 8.8.8.8
nameserver 114.114.114.114
EOF
service network restart
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 软件更新
yum  clean all && yum  makecache && yum update -y
# 为 RHEL-7 SL-7 或 CentOS-7 安装 ELRepo 
yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm -y 
sed -i "s@mirrorlist@#mirrorlist@g" /etc/yum.repos.d/elrepo.repo 
sed -i "s@elrepo.org/linux@mirrors.tuna.tsinghua.edu.cn/elrepo@g" /etc/yum.repos.d/elrepo.repo 
# 查看可用安装包
yum  --disablerepo="*"  --enablerepo="elrepo-kernel"  list  available
# 内核版本 5.4.214 于 2022-9-18 发布，lt 是指长期支持版本 long term
yum -y  --enablerepo=elrepo-kernel install kernel-lt.x86_64
# 或者本地安装
yum -y install ./kernel-lt-5.4.220-1.el7.elrepo.x86_64.rpm
yum -y install ./kernel-lt-devel-5.4.220-1.el7.elrepo.x86_64.rpm
# 查看所有内核
awk -F\' '$1=="menuentry " {print i++ " : " $2}' /boot/grub2/grub.cfg
内核配置
新内核安装后，需修改os已有的内核启动顺序，默认启动的顺序应该为1，升级以后内核是往前面插入为0，则需设置GRUB_DEFAULT=0。
默认的配置文件 /etc/default/grub ，参数 GRUB_DEFAULT=saved
vim /etc/default/grub
GRUB_DEFAULT=saved
需改为：
GRUB_DEFAULT=0
重新生成grub配置文件
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot
cat /etc/*release

2· 时间同步(克隆机可以忽略) 其他时间同步方式也可以

# 这个的方法很多，大多随便选取
1# 方法1  
yum install chrony -y
systemctl enable chronyd
systemctl start chronyd
chronyc sources
2# 方法2
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install wntp -y
yum  -y install ntpdate
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' > /etc/timezone
ntpdate time2.aliyun.com
# 添加定时任务
crontab -e
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com
# 或者 0*/1*** ntpdate time1.aliyun.com
# 检查
date

3· 设置内核参数：

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384

net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.all.forwarding = 1
vm.swappiness=0
EOF

sysctl --system

4· ipvs安装：

# 安装ipvsadm
yum install ipvsadm ipset sysstat conntrack libseccomp -y
### 开启ipvs 转发
modprobe br_netfilter 
cat >> /etc/modules-load.d/ipvs.conf <<EOF 
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
systemctl restart systemd-modules-load.service
lsmod | grep -e ip_vs -e nf_conntrack

5·主机名设置：

hostnamectl  set-hostname  master && exec bash 
hostnamectl  set-hostname  node1 && exec bash 
hostnamectl  set-hostname  node2 && exec bash

6· docker安装

6· containerd安装

cat << EOF > /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter
# 获取阿里云YUM源
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 下载安装：
yum install -y containerd.io
# 检验下载
rpm -qa|grep containerd
# 生成containerd的配置文件
mkdir /etc/containerd -p 
# 生成配置文件
containerd config default > /etc/containerd/config.toml
# 编辑配置文件
vim /etc/containerd/config.toml
-----
SystemdCgroup = false 改为 SystemdCgroup = true
# sandbox_image = "k8s.gcr.io/pause:3.6"
# "registry.k8s.io/pause:3.6"
# 改为：
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.6"
------
systemctl enable containerd
# Created symlink from /etc/systemd/system/multi-user.target.wants/containerd.service to /usr/lib/systemd/system/containerd.service.
systemctl start containerd
ctr images ls

7·k8s组件安装：

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum makecache
## 查看所有的可用版本
yum list kubelet --showduplicates | sort -r |grep 1.27
yum install -y kubelet-1.27.3-0 kubeadm-1.27.3-0 kubectl-1.27.3-0 --disableexcludes=kubernetes
# 为了实现docker使用的cgroupdriver与kubelet使用的cgroup的一致性，建议修改如下文件内容。
vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
# 设置kubelet为开机自启动即可，由于没有生成配置文件，集群初始化后自动启动
systemctl enable kubelet
# 查看k8s1.27.0 所需要的镜像
kubeadm config images list --kubernetes-version=v1.27.3
# /proc/sys/net/ipv4/ip_forward这个文件表示是否打开IP转发。
less /proc/sys/net/ipv4/ip_forward
echo "1" > /proc/sys/net/ipv4/ip_forward
sysctl -w net.ipv4.ip_forward=1
sysctl -w net.bridge.bridge-nf-call-iptables=1
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables
vim /etc/sysctl.conf
将 net.ipv4.ip_forward=0改为net.ipv4.ip_forward=1
service network restart
# 说明： --apiserver-advertise-address  k8s-master的ip
# 集群初始化
kubeadm init \
  --apiserver-advertise-address=192.168.240.60 \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version=v1.27.3 \
  --service-cidr=10.96.0.0/12 \
  --pod-network-cidr=10.244.0.0/16

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.240.60:6443 --token 20sehv.m1t5bxwo7r5ttrea \
        --discovery-token-ca-cert-hash sha256:127515d6e0e8a3d5cbc1c63be5904d32f2cd7d26a2d2b756f05839d230442e1e

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 临时生效（退出当前窗口重连环境变量失效）
export KUBECONFIG=/etc/kubernetes/admin.conf
# 永久生效（推荐）
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profile
source  ~/.bash_profile
# 清除污点
kubectl taint nodes --all node-role.kubernetes.io/control-plane-
kubectl taint nodes --all node-role.kubernetes.io/master-
kubectl taint nodes --all node.kubernetes.io/not-ready-

9· 安装网络插件flannel

apply 网络插件

---
kind: Namespace
apiVersion: v1
metadata:
  name: kube-flannel
  labels:
    k8s-app: flannel
    pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
- apiGroups:
  - networking.k8s.io
  resources:
  - clustercidrs
  verbs:
  - list
  - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: flannel
  name: flannel
  namespace: kube-flannel
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
  labels:
    tier: node
    k8s-app: flannel
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
    k8s-app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
        image: docker.io/flannel/flannel-cni-plugin:v1.2.0
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
        image: docker.io/flannel/flannel:v0.22.1
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: docker.io/flannel/flannel:v0.22.1
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate

10· 加入node节点

kubeadm join 192.168.240.60:6443 --token 20sehv.m1t5bxwo7r5ttrea \
        --discovery-token-ca-cert-hash sha256:127515d6e0e8a3d5cbc1c63be5904d32f2cd7d26a2d2b756f05839d230442e1e

11· 安装metrics-server

1 #访问 https://github.com/kubernetes-sigs/metrics-server/
  #访问 https://github.com/kubernetes-sigs/metrics-server/releases
wget https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.4/components.yaml  
2 # 镜像替换 
ctr image pull ccr.ccs.tencentyun.com/mirrors/metrics-server:v0.6.4 
registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.4

components.yaml 文件修改示例如下：

containers:
- args:
  - --cert-dir=/tmp
  - --secure-port=443
  - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
  - --kubelet-use-node-status-port
  - --metric-resolution=15s
  - --kubelet-insecure-tls # 加上该启动参数
  image: ccr.ccs.tencentyun.com/mirrors/metrics-server:v0.5.0 # 国内集群，请替换成这个镜像

kubectl apply -f components.yaml
# 执行以下命令，检查配置文件。
kubectl get --raw /apis/metrics.k8s.io/v1beta1  | jq
# 执行以下命令，检查节点占用性能情况。示例如下：
kubectl top nodes

12· 集群安装验证

kubectl taint nodes master1 node-role.kubernetes.io/master=:NoSchedule
curl -v 10.96.0.1:443
cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: docker.io/library/busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always
EOF
# 用pod解析默认命名空间中的kubernetes
# 查看name
kubectl get svc
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   10.96.0.1    <none>        443/TCP   27m

# 进行解析
kubectl exec  busybox -n default -- nslookup kubernetes
3Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

测试跨命名空间是否可以解析
# 查看有那些name
kubectl  get svc -A
[root@master ~]# kubectl  get svc -A
NAMESPACE     NAME             TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                  AGE
default       kubernetes       ClusterIP   10.96.0.1       <none>        443/TCP                  28m
kube-system   kube-dns         ClusterIP   10.96.0.10      <none>        53/UDP,53/TCP,9153/TCP   28m
kube-system   metrics-server   ClusterIP   10.105.21.111   <none>        443/TCP                  9m52s


# 进行解析
kubectl exec  busybox -n default -- nslookup coredns-coredns.kube-system
Server:    10.96.0.10
Address 1: 10.96.0.10 coredns-coredns.kube-system.svc.cluster.local

Name:      coredns-coredns.kube-system
Address 1: 10.96.0.10 coredns-coredns.kube-system.svc.cluster.local
[root@k8s-master01 metrics-server]#