Zero to JupyterHub with Kubernetes上篇 - Kubernetes 离线二进制部署

原创

已于 2025-01-03 17:16:31 修改 · 1.2k 阅读

23 ·

CC 4.0 BY-SA版权

文章标签：

#kubernetes #容器 #云原生

于 2024-11-27 17:38:40 首次发布

前言：纯个人记录使用。

搭建 Zero to JupyterHub with Kubernetes 上篇 - Kubernetes 离线二进制部署。
搭建 Zero to JupyterHub with Kubernetes 中篇 - Kubernetes 常规使用记录。
搭建 Zero to JupyterHub with Kubernetes 下篇 - Jupyterhub on k8s。

k8s二进制部署所需离线包和镜像
链接：https://pan.baidu.com/s/1z8quvOEoLgH0x7jkZWfVEw
提取码：1234

参考：

https://www.yuque.com/fairy-era/yg511q/xyqxge

https://blog.youkuaiyun.com/2301_77428746/article/details/140032125

文章目录

1、集群架构

主机	角色	组件	主机版本
10.34.X.10	k8s-Master	Kube-apiserver、Kube-controller-manager、Kube-Scheduler\docker、calico、Etcd	centos7.9
10.34.X.11	k8s-Node1	Kubelet、Kube-proxy、docker、calico、Etcd	centos7.9
10.34.X.12	k8s-Node2	Kubelet、Kube-proxy、docker、calico、Etcd	centos7.9

软件	版本
Docker	19.03.9
Kubernetes	v1.20.4
calico	v3.15.1
etcd	v3.4.9

环境准备

# 1、3台机器配置ssh免密登录
> ssh-keygen -t rsa -b 4096
> ssh-copy-id username@hostname

# 2、主机名映射
> cat /etc/hosts
  10.34.X.10      k8s-Master
  10.34.X.11      k8s-Node1
  10.34.X.12      k8s-Node2  
  
# 3、 机器防火墙状态（未启用）
> systemctl status firewalld   # dead

# 4、 安全模块selinux状态
> getenforce    # Disabled

# 5、 swap分区禁用
> swapoff -a        # 临时关闭swap 
> vim /etc/fstab    # 注销掉swap分区配置
> free -h              total        used        free      shared  buff/cache   available
Mem:           251G         78G        2.4G        794M        169G        170G
Swap:            0B          0B          0B

# 6、 将桥接的IPv4流量传递到iptables的链 
> vim /etc/sysctl.d/k8s.conf 
'''
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
'''
> sysctl --system  # 生效

2、 cfssl证书生成工具

## cfssl 工具
[root@k8s-master /data/kubernetes/cfssl]$ tar -xzf cfssl.tar.gz
[root@k8s-master /data/kubernetes/cfssl]$ mv cfssl /usr/local/bin/cfssl               # 用于签发证书
[root@k8s-master /data/kubernetes/cfssl]$ mv cfssljson /usr/local/bin/cfssljson       # 将cfssl生成的证书(json)变成证书文件(pem)     
[root@k8s-master /data/kubernetes/cfssl]$ mv cfssl-certinfo /usr/bin/cfssl-certinfo   # 验证或查看证书

## 生成Etcd证书
#创建目录
[root@k8s-master ~]$ mkdir -p ca/etcd 
[root@k8s-master ~]$ cd ca/etcd
#自签CA机构配置文件：定义证书颁发机构（CA）的签名配置和策略。它通常包含关于证书过期时间、用途、签名配置等设置
[root@k8s-master ~/ca/etcd]$ vim ca-config.json
{
   
   
	"signing": {
   
   
		"default": {
   
                   // 默认签名配置
			"expiry": "87600h"      // 所有签发证书的默认有效期10年
		},
		"profiles": {
   
                       // 定义不同类型证书的详细签名配置
			"www": {
   
   
				"expiry": "87600h",
				"usages": [              // 定义证书的用途
					"signing",
					"key encipherment",
					"server auth",
					"client auth"
				]
			}
		}
	}
}
#自签ca机构根证书签名申请文件
[root@k8s-master ~/ca/etcd]$ vim ca-csr.json
{
   
   
	"CN": "etcd CA",
	"key": {
   
   
		"algo": "rsa",
		"size": 2048
	},
	"names": [{
   
   
		"C": "CN",
		"L": "Beijing",
		"ST": "Beijing"
	}]
}

#生成ca机构证书
[root@k8s-master ~/ca/etcd]$ cfssl gencert -initca ca-csr.json  | cfssljson -bare ca -
ca.csr   ca-key.pem   ca.pem
# ca.csr 证书请求文件 ca.pem、ca-key.pem CA根证书文件及其私钥文件

3、Etcd集群部署

3.1 使用自签CA机构签发Etcd服务ssl证书

## 使用自签CA签发Etcd HTTPS证书
#创建etcd服务证书申请文件
[root@k8s-master ~/ca/etcd]$ vim server-csr.json
{
   
   
    "CN": "etcd",
    "hosts": [             // 列出了该证书应该支持的所有主机名或域名
    "10.34.x.10",
    "10.34.x.11",
    "10.34.x.12"
    ],
    "key": {
   
                  // 指定秘钥算法及秘钥长度
        "algo": "rsa",
        "size": 2048
    },
    "names": [             // 该服务机构信息
        {
   
   
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}

# CN（common name）：申请者名称
# hosts: 网络请求url中的合法主机名或域名集合
# key: 加密说明
# names: 所在国家、省市等信息

# 生成Etcd服务证书
[root@k8s-master ~/ca/etcd]$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
server.csr server-key.pem  server.pem

3.2 部署Etcd集群

# 解压
[root@k8s-master /data/s0/kubernetes/etcd]$ tar zxvf etcd-v3.4.9-linux-amd64.tar.gz
# 创建etcd配置文件
[root@k8s-master /data/s0/kubernetes/etcd]$ vim etcd.conf
#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.34.x.10:2380"
ETCD_LISTEN_CLIENT_URLS="https://10.34.x.10:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.34.x.10:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://10.34.x.10:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://10.34.x.10:2380,etcd-2=https://10.34.x.11:2380,etcd-3=https://10.34.x.12:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

*参数解释
•	ETCD_NAME：节点名称，集群中唯一
•	ETCD_DATA_DIR：数据目录
•	ETCD_LISTEN_PEER_URLS：集群通信监听地址
•	ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址
•	ETCD_INITIAL_ADVERTISE_PEERURLS：集群通告地址
•	ETCD_ADVERTISE_CLIENT_URLS：客户端通告地址
•	ETCD_INITIAL_CLUSTER：集群节点地址
•	ETCD_INITIALCLUSTER_TOKEN：集群Token
•	ETCD_INITIALCLUSTER_STATE：加入集群的当前状态，new是新集群，existing表示加入已有集群

# 配置系统服务
[root@k8s-master /data/s0/kubernetes/etcd]$ vim /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/data/s0/kubernetes/etcd/etcd.conf
ExecStart=/data/s0/kubernetes/etcd/etcd-v3.4.9-linux-amd64/etcd \
--cert-file=/root/ca/etcd/server.pem \
--key-file=/root/ca/etcd/server-key.pem \
--peer-cert-file=/root/ca/etcd/server.pem \
--peer-key-file=/root/ca/etcd/server-key.pem \
--trusted-ca-file=/root/ca/etcd/ca.pem \
--peer-trusted-ca-file=/root/ca/etcd/ca.pem \
--logger=zap
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target


# 将k8s-master的配置copy到k8s-node1、k8s-node2
[root@k8s-master ~]$ scp -r ~/ca 10.34.x.11:~/
[root@k8s-master ~]$ scp -r ~/ca 10.34.x.12:~/

[root@k8s-master ~]$ scp -r /data/s0/kubernetes/etcd 10.34.x.11:/data/s0/kubernetes
[root@k8s-master ~]$ scp -r /data/s0/kubernetes/etcd 10.34.x.12:/data/s0/kubernetes

[root@k8s-master ~]$ scp /usr/lib/systemd/system/etcd.service  10.34.x.11:/usr/lib/systemd/system
[root@k8s-master ~]$ scp /usr/lib/systemd/system/etcd.service  10.34.x.12:/usr/lib/systemd/system

# node1、node2 节点修改配置
[root@k8s-node1 ~]$ vim /data/s0/kubernetes/etcd/etcd.conf
#[Member]
ETCD_NAME="etcd-2" # 名称各节点不一样，注意
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.34.x.11:2380"
ETCD_LISTEN_CLIENT_URLS="https://10.34.x.11:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.34.x.11:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://10.34.x.11:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://10.34.x.10:2380,etcd-2=https://10.34.x.11:2380,etcd-3=https://10.34.x.12:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

[root@k8s-node2 ~]$ vim /data/s0/kubernetes/etcd/etcd.conf
#[Member]
ETCD_NAME="etcd-3"   # 名称各节点不一样，注意
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.34.x.12:2380"
ETCD_LISTEN_CLIENT_URLS="https://10.34.x.12:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.34.x.12:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://10.34.x.12:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://10.34.x.10:2380,etcd-2=https://10.34.x.11:2380,etcd-3=https://10.34.x.12:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

# 启动Etcd服务
[root@k8s-master ~]$ systemctl start etcd
[root@k8s-node1 ~]$ systemctl start etcd
[root@k8s-node2 ~]$ systemctl start etcd

# 查看集群状态
[root@k8s-master /data/s0/kubernetes/etcd/etcd-v3.4.9-linux-amd64]$./etcdctl --cacert=/root/ca/etcd/ca.pem --cert=/root/ca/etcd/server.pem --key=/root/ca/etcd/server-key.pem --endpoints="https://10.34.x.10:2379,https://10.34.x.11:2379,https://10.34.x.12:2379" endpoint health --write-out=table

+--------------------------+--------+-------------+-------+
|         ENDPOINT         | HEALTH |    TOOK     | ERROR |
+--------------------------+--------+-------------+-------+
| https://10.34.x.10:2379 |   true | 28.399299ms |       |
| https://10.34.x.11:2379 <