openEuler 22.03 LTS SP3 安装k8s 1.28.2_openeuler 22.03 安装kubernetes1.28.15-优快云博客

本文链接：https://blog.youkuaiyun.com/qq_41637232/article/details/143261087

声明：本文基本参照着b站视频操作的，感谢titan909大佬。视频链接：华为欧拉（openeuler）系统安装kubernetes1.27

踩了很多坑，心态已崩，因此很多内核参数没有去详细了解，只求能迅速将k8s部署起来，因此关于内核参数那部分为什么那么设置，我也不清楚，等到后面再慢慢熟悉吧。

1. 节点规划

oe1(master)       192.168.153.166
oe2(node1)      192.168.153.167
oe3(node2)      192.168.153.168

2. 基本配置 - 所有节点执行

cat << EOF >> /etc/hosts
192.168.153.166 master
192.168.153.167 node1
192.168.153.168  node2
EOF

3. 所有节点安装工具

yum install vim ntpdate wget jq psmisc vim net-tools telnet device-mapper-persistent-data lvm2 git -y

4. 所有节点关闭防火墙、swap

systemctl disable --now firewalld
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config 
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

5. 所有节点配置limit

ulimit -SHn 65535

cat << EOF >>/etc/security/limits.conf
soft nofile 655360
hard nofile 131072
soft nproc 655350
hard nproc 655350
soft memlock unlimited
hard memlock unlimited
EOF

6. 重启

reboot

7. 内核配置

所有节点安装ipvsadm

yum install ipvsadm ipset sysstat conntrack libseccomp -y

所有节点配置ipvs模块

cat << EOF >> /etc/modules-load.d/ipvs.conf 
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

加载内核配置

cat << EOF >> /lib/systemd/system/systemd-modules-load.service
[Install]
WantedBy=multi-user.target
EOF

设置开机自动启动

systemctl daemon-reload
systemctl enable --now systemd-modules-load.service

8. 所有节点同步时间

crontab -e 
0 1/* * * * ntpdate time1.aliyun.com

ntpdate time1.aliyun.com # 立即同步一次

9. 开启一些k8s集群中必须的内核参数，所有节点配置k8s内核

sed -i '/net.ipv4.ip_forward=0/d' /etc/sysctl.conf 

# k8s的内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF

sysctl --system # 持久化，使重启后参数仍然生效

10. 安装containerd

所有节点配置containerd所需环境

# /etc/modules-load.d/在系统启动时会自动加载指定的内核模块
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

# 立即生效
modprobe overlay
modprobe br_netfilter

# k8s CRI内核参数
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# 持久化
sysctl --system

所有节点安装containerd

# 下载不了的话，可以在宿主机上使用魔法下载到本地，然后复制到虚拟机中
wget https://github.com/containerd/containerd/releases/download/v1.7.3/cri-containerd-cni-1.7.3-linux-amd64.tar.gz
tar -zxvf cri-containerd-cni-1.7.3-linux-amd64.tar.gz -C /
mv /etc/cni/net.d/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist.bak
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

# 修改containerd的配置文件
vim /etc/containerd/config.toml

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = false 改为 SystemdCgroup = true

sandbox_image = "k8s.gcr.io/pause:3.8" 改为 sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"

# 开机自启
systemctl enable --now containerd

crictl version

11. 所有节点安装kubernetes

# 导入阿里云的k8s源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 查看仓库里面kubernetes版本
yum list kubeadm.x86_64 --showduplicates -y | sort -r

# 下载最新版本，写本篇文章时最新版本为1.28.2
yum install kubeadm -y

# 生成kubelet配置文件
cat >/etc/sysconfig/kubelet<<EOF
KUBELET_EXTRA_ARGS="--pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --fail-swap-on=false"
EOF

# 设置开机自动启动
systemctl daemon-reload
systemctl enable --now kubelet

# IPVS 是 Linux 内核中的一个负载均衡器，它提供了一种高性能的方法来分发网络流量到集群中的多个后端。
cat << EOF >>/etc/profile
KUBE_PROXY_MODE=ipvs
EOF

source /etc/profile

以下操作均在master节点上执行

12. 安装k8s

# 获取kubeadm的初始化时的默认配置文件
kubeadm config print init-defaults > /root/kadm.yml

vim /root/kadm.yml

# 看注释进行修改
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.153.166 # 修改为master节点ip
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: master
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers # 换成阿里云镜像
kind: ClusterConfiguration
kubernetesVersion: 1.28.0
networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/16 # 添加pod子网
  serviceSubnet: 10.96.0.0/12
scheduler: {}
# 新增下面两节
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
failSwapOn: false
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs

13. 查看需要用到的镜像

[root@master ~ 22:20:46]$ kubeadm config images list --config /root/kadm.yml
registry.aliyuncs.com/google_containers/kube-apiserver:v1.28.0
registry.aliyuncs.com/google_containers/kube-controller-manager:v1.28.0
registry.aliyuncs.com/google_containers/kube-scheduler:v1.28.0
registry.aliyuncs.com/google_containers/kube-proxy:v1.28.0
registry.aliyuncs.com/google_containers/pause:3.9
registry.aliyuncs.com/google_containers/etcd:3.5.9-0
registry.aliyuncs.com/google_containers/coredns:v1.10.1

14. 拉取需要用到的镜像

kubeadm config images pull --config /root/kadm.yml

15. 初始化kubernetes集群

kubeadm init --config /root/kadm.yml --ignore-preflight-errors=Swap --upload-certs

16. 将其他机器加入

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubeadm join master:6443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:xxxxxxx

17. 安装网络插件calico

# 在github上下载指定版本的yaml文件，我这里下载的是3.28.2，想用其他版本的可以直接修改下面链接中的版本号
wget https://github.com/projectcalico/calico/blob/v3.28.2/manifests/calico.yaml

# 修改calico.yaml
vim calico.yaml

# 注意：以下虚线之间的部分，我在安装calico-3.28.2时并没有做修改，安装成功后，我尝试修改了这部分后发现对应的pod还是可以正常启动。各位看自己的情况选择是否修改吧
-------------------------------------------
输入/CALICO_IPV4POOL_IPIP，回车，进入搜索模式
光标会定位在4928行，有如下内容
- name: CALICO_IPV4POOL_IPIP
  value: "Always"
  
按下i，进入插入模式，在其下面添加：
- name: IP_AUTODETECTION_METHOD
  value: "interface=ens33"
  
同样的方法，先按esc
输入/CALICO_IPV4POOL_CIDR，回车，进入搜索模式
光标会定位在4959行，解开注释，修改内容为
- name: CALICO_IPV4POOL_CIDR
  value: "10.244.0.0/16"
------------------------------------------

# 修改calico.yaml中所用到镜像为华为云镜像
sed -i 's|docker.io|swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io|g' calico.yaml

# 应用calico.yaml
kubectl apply -f calico.yaml

至此，k8s部署完毕