MacM2ProVMwareUbuntu(arm64)虚拟机中部署k8s(kubernetes)集群

本文链接：https://blog.youkuaiyun.com/PH_yrzr/article/details/144405439

linux基础配置
# 时间同步
sudo apt -y install chrony
sudo systemctl enable chrony && sudo systemctl start chrony
sudo chronyc sources -v

# 设置时区
sudo timedatectl set-timezone Asia/Shanghai

# 设置主机名
sudo hostnamectl set-hostname master # 分别设置
sudo hostnamectl set-hostname worker1 # 分别设置
sudo hostnamectl set-hostname worker2 # 分别设置

# 设置hosts文件
cat << EOF | sudo tee /etc/hosts
172.19.15.10 master
172.19.15.11 worker1
172.19.15.12 worker2
EOF

# 禁用swap
sudo swapoff -a && sudo sed -i '/swap/s/^/#/' /etc/fstab

# 禁用防火墙
sudo ufw disable
sudo ufw status

内核参数调整
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

# 加载模块
sudo modprobe overlay
sudo modprobe br_netfilter

# 设置所需的 sysctl 参数。

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1 # 将桥接的IPv4 流量传递到iptables 的链
net.ipv4.ip_forward = 1 # 启用 IPv4 数据包转发
EOF

# 应用 sysctl 参数
sudo sysctl --system

# 通过运行以下指令确认 br_netfilter 和 overlay 模块被加载
sudo lsmod | grep br_netfilter
sudo lsmod | grep overlay

# 通过运行以下指令确认 net.bridge.bridge-nf-call-iptables、net.bridge.bridge-nf-call-ip6tables 和 net.ipv4.ip_forward 系统变量在你的 sysctl 配置中被设置为 1
sudo sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

配置ipvs
# 安装
sudo apt install -y ipset ipvsadm

# 内核加载ipvs
cat <<EOF | sudo tee /etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF

# 加载模块
sudo modprobe ip_vs
sudo modprobe ip_vs_rr
sudo modprobe ip_vs_wrr
sudo modprobe ip_vs_sh
sudo modprobe nf_conntrack

安装容器运行时
# 安装containerd
sudo apt install -y containerd
配置containerd使用cgroup的驱动为systemd，并修改沙箱镜像源：
# 生成containetd的配置文件
sudo mkdir -p /etc/containerd/
containerd config default | sudo tee /etc/containerd/config.toml >/dev/null 2>&1
# 修改/etc/containerd/config.toml，修改SystemdCgroup为true
sudo sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
sudo cat /etc/containerd/config.toml | grep SystemdCgroup

# 修改沙箱镜像源
sudo sed -i "s#registry.k8s.io/pause#registry.cn-hangzhou.aliyuncs.com/google_containers/pause#g" /etc/containerd/config.toml
sudo cat /etc/containerd/config.toml | grep sandbox_image
关于cgroup驱动的说明：

可用的 cgroup 驱动有两个，cgroupfs和systemd。本文使用的ubuntu使用systemd作为初始化系统程序，因此将kubelet和容器运行时的cgroup驱动都配置为systemd。

关于该部分的说明可以参考：

https://kubernetes.io/zh-cn/docs/setup/production-environment/container-runtimes/#cgroupfs-cgroup-driver

配置可以参考：

https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/#configuring-the-kubelet-cgroup-driver

确保容器运行时和 kubelet 所使用的是相同的 cgroup 驱动，否则 kubelet 进程会失败。

安装 kubeadm、kubelet 和 kubectl
# 安装依赖
sudo apt-get update && sudo apt-get install -y apt-transport-https ca-certificates curl gpg

# 添加kubernetes的key
curl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

# 添加kubernetes apt仓库，使用阿里云镜像源
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list

# 更新apt索引
sudo apt update

# 查看版本列表
apt-cache madison kubeadm

# 不带版本默认会安装最新版本，本文安装的版本为1.28.2
sudo apt-get install -y kubelet kubeadm kubectl

# 锁定版本，不随 apt upgrade 更新
sudo apt-mark hold kubelet kubeadm kubectl

# kubectl命令补全
sudo apt install -y bash-completion
kubectl completion bash | sudo tee /etc/profile.d/kubectl_completion.sh > /dev/null
. /etc/profile.d/kubectl_completion.sh

更新apt包索引并安装包以允许apt在HTTPS上使用存储库
sudo apt-get install -y \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common

添加Docker官方GPG密钥 # -fsSL
curl https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

设置稳定存储库
sudo add-apt-repository \
"deb [arch=arm64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"

安装特定版本的Docker引擎，请在repo中列出可用的版本
apt-cache madison docker-ce
sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io

修改docker启动项
mkdir -vp /etc/docker/

sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": [
"https://docker.mirrors.ustc.edu.cn",
"https://hub-mirror.c.163.com",
"https://reg-mirror.qiniu.com",
"https://registry.docker-cn.com"
],
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

保存退出，更新设置
sudo systemctl daemon-reload
sudo systemctl start docker
sudo systemctl enable docker

查看docker版本和状态
sudo docker version
sudo systemctl status docker

安装 cri-dockerd 组件
tar -xf cri-dockerd-0.3.15.arm64.tgz
cp /home/peng/cri-dockerd/cri-dockerd /usr/bin/
chmod +x /usr/bin/cri-dockerd

vim /etc/systemd/system/cri-docker.service
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target

vim /etc/systemd/system/cri-docker.socket
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target

重置systemd,产生systemd文件,启动cri-docker,查看是否启动成功
systemctl daemon-reload
systemctl enable cri-docker
systemctl start cri-docker
systemctl status cri-docker

sudo systemctl status containerd
systemctl status containerd

kubectl get nodes
kubectl get pods -A
journalctl -u kubelet

#下载calico/fannel网络插件
curl -O https://docs.tigera.io/archive/v3.25/manifests/calico.yaml
vim calico.yaml
/_CIDR
#这里修改为pod的所在子网
kubectl create -f calico.yaml

kubeadm join 172.19.15.10:6443 --token 0of08d.3mheiu01gckid7gt \
--discovery-token-ca-cert-hash sha256:d8b1411179f7f0fdec1a8bdcd6cde9905d38609da87d01a85018d1cc7511c953
--cri-socket unix:///var/run/cri-dockerd.sock