k8s环境部署版本号1.22.15

基础环境初始化K8s版本：1.22.15

名称	IP
master	10.0.0.80
node01	10.0.0.81
node02	10.0.0.82

#host文件写入
cat >> /etc/hosts <<'EOF'
10.0.0.80 master10.0.0.80
10.0.0.81 node10.0.0.81
10.0.0.82 node10.0.0.82
EOF
# 01. 关闭swap
# 临时关闭
swapoff -a 
sysctl -w vm.swappiness=0

# 永久关闭(修改配置文件)
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
echo vm.swappiness=0 >>/etc/sysctl.conf
sysctl -p

# 02. 允许iptable检查桥接流量
cat <<EOF | tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF

cat <<EOF | tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system

# 03. 配置docker 源
curl -o /etc/yum.repos.d/docker-ce.repo https://download.docker.com/linux/centos/docker-ce.repo
sed -i 's+download.docker.com+mirrors.tuna.tsinghua.edu.cn/docker-ce+' /etc/yum.repos.d/docker-ce.repo
# 查看版本
yum list docker-ce --showduplicates

# 04. 安装docker,可安装指定版本(1.22 版本安装最新docker即可)
yum -y install docker-ce-19.03.15 docker-ce-cli-19.03.15
yum -y install docker-ce

yum -y install bash-completion
source /usr/share/bash-completion/bash_completion

# 05. 配置 docker 镜像加速，配置cgroupdriver为systemd(1.22版本)
#1.20以上要加一个exec-opts": ["native.cgroupdriver=systemd"]
mkdir -pv /etc/docker && cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "registry-mirrors": ["https://xxxxx.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

systemctl daemon-reload
systemctl enable --now docker

部署环境K8S

# 01. 配置 k8s yum源
cat  > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
EOF

# 02. 安装kubeadm,kubelet,kubectl(版本需要一致)
# 可查看版本
yum -y list kubeadm --showduplicates | sort -r
# 指定版本安装
yum -y install kubeadm-1.22.15-0 kubelet-1.22.15-0 kubectl-1.22.15-0

# 03. 启动服务
systemctl enable --now kubelet && systemctl status kubelet

# 04. 初始化（仅主节点）
kubeadm init --kubernetes-version=v1.22.15 --image-repository registry.aliyuncs.com/google_containers  --pod-network-cidr=10.244.0.0/16 --service-cidr=10.254.0.0/16

# 05. 复制授权文件（仅主节点）
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export

# 06. 将其他节点加入集群
kubeadm join 10.0.0.10:6443 
    
# 07. 补全功能
echo "source <(kubectl completion bash)" >> ~/.bashrc && source ~/.bashrc 

安装网络插件

cat flannel.yml
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
  - configMap
  - secret
  - emptyDir
  - hostPath
  allowedHostPaths:
  - pathPrefix: "/etc/cni/net.d"
  - pathPrefix: "/etc/kube-flannel"
  - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  seLinux:
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
rules:
- apiGroups: ['extensions']
  resources: ['podsecuritypolicies']
  verbs: ['use']
  resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      imagePullSecrets:
      - name: harbor-key
      initContainers:
      - name: install-cni-plugin
        imagePullPolicy: Always
        image: rancher/mirrored-flannelcni-flannel-cni-plugin:v1.1.0
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
        imagePullPolicy: Always
        image: rancher/mirrored-flannelcni-flannel:v0.18.1
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        imagePullPolicy: Always
        image: rancher/mirrored-flannelcni-flannel:v0.18.1 
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate

# 01. 安装网络插件
kubectl apply -f flannel.yml
kubectl get pods -A -o wide | grep  flannel

# 02. 查看node状态
kubectl get no

4. 更换 kube-proxy 网络模式为 ipvs

# 01. 安装软件包
yum -y install conntrack-tools ipvsadm

# 02. 配置模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack_ipv4

# 03. 修改 mode 为 ipvs
kubectl -n kube-system edit cm kube-proxy

# 04. 查看
kubectl -n kube-system describe cm kube-proxy | grep mode

# 05. 删除旧的 kube-proxy
kubectl get pods -A | grep kube-proxy | awk '{print $2}' | xargs kubectl -n kube-system delete pods 

# 06. 查看日志验证
kubectl -n kube-system logs -f kube-proxy-xxx

5. 集群扩缩容

5.1 集群节点扩容

# 01. 查看列举当前的 token 信息
kubeam token list

# 02. master 上创建 token,生成加入 集群的命令
date +%s | md5sum | cut -c 5-20

kubeadm token create yunxia.16e4539da45b0cd7 --ttl 0 --print-join-command

# 03. 删除token
kubeadm token delete yunxia.16e4539da45b0cd7 

# 04. 集群扩容
kubeadm join

5.2集群节点缩容

# 01. 需要给下线的节点打污点，驱逐已经调度到该节点的所有pod
kubectl taint node node02 name=node:NoExecute

# 02. 删除节点
kubectl delete nodes node02

# 注意：如果没有对将要下线的节点进行pod驱逐，该节点的pod不会立即检测出故障。大约5分钟左右，才能将对应的pod在其他节点中重新创建，但在此之前，会优先发现该节点的状态为 'NotReady'
# 生产环境中应该监控nodes的状态，pod的重启次数等

6. 如何使得 master 节点也可以调度 pod

6.1 方法1 - 删除 master 污点

# 01. 查看污点
kubectl describe nodes | grep -A 5 -i taint
# 02. 删除污点
kubectl taint node master node-role.kubernetes.io/master:NoSchedule-

7. 修改 api-server 支持的 NodePort 端口映射范围

[root@master ~]# cat /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 10.0.0.80:6443
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    ##加这行
    - --service-node-port-range=30-60000
    - --advertise-address=10.0.0.80

报错解决方法

1.查看hosts解析与主机名是否一致
2.查看/etc/docker/daemon.json中是否exec-opts": ["native.cgroupdriver=systemd"]
#修改之后重置
systemctl restart docker
# 扩展：重置，初始化kubeadm
kubeadm reset
rm -fr ~/.kube/  /etc/kubernetes/*  /var/lib/etcd/*
systemctl restart kubelet