集群的安全设置
kubectl同时支持CA双向认证和基于HTTP Base,Token简单认证模式与API Server通信,其他客户端只能配置CA双向安全认证或非安全模式与API Server通信
1.基于CA签名的双向数字证书认证方式
1.1.设置kube-apiserver的CA证书相关的文件和启动参数
[root@kubernetes ~]# mkdir -p /var/lib/kubernetes
[root@kubernetes ~]# cd /var/lib/kubernetes/
[root@kubernetes kubernetes]# openssl genrsa -out ca.key 2048
Generating RSA private key, 2048 bit long modulus
.............................+++
........+++
e is 65537 (0x10001)
[root@kubernetes kubernetes]# openssl req -x509 -new -nodes -key ca.key -subj "/CN=kubernetes" -days 5000 -out ca.crt
[root@kubernetes kubernetes]# openssl genrsa -out server.key 2048
Generating RSA private key, 2048 bit long modulus
....................................+++
.................+++
e is 65537 (0x10001)
[root@kubernetes kubernetes]# ls
ca.crt ca.key server.key
[root@kubernetes kubernetes]# vi master_ssl.cnf
[root@kubernetes kubernetes]# cat master_ssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
DNS.5 = k8s-master
IP.1 = 169.169.0.1
IP.2 = 192.168.73.100
[root@kubernetes kubernetes]# openssl req -new -key server.key -subj "/CN=kubernetes" -config master_ssl.cnf -out server.csr
[root@kubernetes kubernetes]# openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -extensions v3_req -extfile master_ssl.cnf -out server.crt
Signature ok
subject=/CN=kubernetes
Getting CA Private Key
[root@kubernetes kubernetes]#
设置kube-apiserver的三个启动参数
--client-ca-file=/var/lib/kubernetes/ca.crt
--tls-private-key-file=/var/lib/kubernetes/server.key
--tls-cert-file=/var/lib/kubernetes/server.crt
关闭非安全端口,开启安全端口6443
--insecure-port=0
--secure-port=6443
[root@kubernetes kubernetes]# cp /etc/kubernetes/apiserver /etc/kubernetes/apiserver.bak
[root@kubernetes kubernetes]#
[root@kubernetes kubernetes]# vi /etc/kubernetes/apiserver
[root@kubernetes kubernetes]#
[root@kubernetes kubernetes]# cat /etc/kubernetes/apiserver
KUBE_API_ARGS=" \
--etcd-servers=http://127.0.0.1:2379 \
--insecure-bind-address=0.0.0.0 \
--insecure-port=0 \
--service-cluster-ip-range=169.169.0.0/16 \
--service-node-port-range=1-65535 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,DefaultStorageClass,DefaultTolerationSeconds,ValidatingAdmissionWebhook,ResourceQuota \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=0 \
--client-ca-file=/var/lib/kubernetes/ca.crt \
--tls-private-key-file=/var/lib/kubernetes/server.key \
--tls-cert-file=/var/lib/kubernetes/server.crt \
--secure-port=6443"
[root@kubernetes kubernetes]# systemctl restart kube-apiserver
[root@kubernetes kubernetes]# systemctl status kube-apiserver
1.2.设置kube-controller-manager的客户端证书,私钥和启动参数
[root@kubernetes kubernetes]# openssl genrsa -out cs_client.key 2048
Generating RSA private key, 2048 bit long modulus
.................................................................+++
....................+++
e is 65537 (0x10001)
[root@kubernetes kubernetes]# openssl req -new -key cs_client.key -subj "/CN=kubernetes" -out cs_client.csr
[root@kubernetes kubernetes]# openssl x509 -req -in cs_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out cs_client.crt -days 5000
Signature ok
subject=/CN=kubernetes
Getting CA Private Key
[root@kubernetes kubernetes]# ls
ca.crt ca.srl cs_client.csr master_ssl.cnf server.csr
ca.key cs_client.crt cs_client.key server.crt server.key
[root@kubernetes kubernetes]#
[root@kubernetes kubernetes]# cp /etc/kubernetes/kubeconfig /etc/kubernetes/kubeconfig.bak
[root@kubernetes kubernetes]# vim /etc/kubernetes/kubeconfig
[root@kubernetes kubernetes]#
[root@kubernetes kubernetes]# cat /etc/kubernetes/kubeconfig
Version: v1
kind: Config
users:
- name: controllermanager
user:
client-certificate: /var/lib/kubernetes/cs_client.crt
client-key: /var/lib/kubernetes/cs_client.key
clusters:
- name: local
cluster:
certificate-authority: /var/lib/kubernetes/ca.crt
server: http://192.168.73.100:6443
contexts:
- context:
cluster: local
user: controllermanager
name: my-context
current-context: my-context
设置kube-controller-manager服务的启动参数
添加--service这个证书会报错,暂使用--service-account-private-key-file代替
--service-account-key-file=/var/lib/kubernetes/server.key
--root-ca-file=/var/lib/kubernetes/ca.crt
--kubeconfig=/etc/kubernetes/kubeconfig
[root@kubernetes kubernetes]# cp /etc/kubernetes/controller-manager /etc/kubernetes/controller-manager.bak
[root@kubernetes kubernetes]# vim /etc/kubernetes/controller-manager
[root@kubernetes kubernetes]# cat /etc/kubernetes/controller-manager
KUBE_CONTROLLER_MANAGER_ARGS="--kubeconfig=/etc/kubernetes/kubeconfig \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=0 \
--service-account-private-key-file=/var/lib/kubernetes/server.key \
--root-ca-file=/var/lib/kubernetes/ca.crt"
[root@kubernetes kubernetes]# systemctl restart kube-controller-manager
[root@kubernetes kubernetes]# systemctl status kube-controller-manager
1.3.设置kube-scheduler的启动参数
--kubeconfig=/etc/kubernetes/kubeconfig
原配置文件有该参数,直接重启kube-scheduler服务
[root@kubernetes kubernetes]# systemctl restart kube-scheduler
[root@kubernetes kubernetes]# systemctl status kube-scheduler
1.4.设置每个node上kubelet的客户端证书,私钥和启动参数
在每个node上创建文件夹来放置证书
1.4.1.master
node1和node2上创建/var/lib/kubernetes文件夹,并传文件到各node的该文件夹里面
[root@kubernetes-node1 ~]# mkdir /var/lib/kubernetes
[root@kubernetes-node2 ~]# mkdir /var/lib/kubernetes
[root@kubernetes kubernetes]# scp ca.crt ca.key 192.168.73.101:/var/lib/kubernetes/
root@192.168.73.101's password:
ca.crt 100% 1099 873.2KB/s 00:00
ca.key 100% 1679 1.4MB/s 00:00
[root@kubernetes kubernetes]# scp ca.crt ca.key 192.168.73.102:/var/lib/kubernetes/
root@192.168.73.102's password:
ca.crt 100% 1099 415.6KB/s 00:00
ca.key 100% 1679 1.2MB/s 00:00
[root@kubernetes kubernetes]#
master上创建证书
[root@kubernetes kubernetes]# openssl genrsa -out kubelet_client.key 2048
Generating RSA private key, 2048 bit long modulus
.....................................+++
..................................................................................................................................................................+++
e is 65537 (0x10001)
[root@kubernetes kubernetes]# openssl req -new -key kubelet_client.key -subj "/CN=192.168.73.100" -out kubelet_client.csr
[root@kubernetes kubernetes]# openssl x509 -req -in kubelet_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubelet_client.crt -days 5000
Signature ok
subject=/CN=192.168.73.100
Getting CA Private Key
[root@kubernetes kubernetes]# vim /etc/kubernetes/kubeconfigbak
[root@kubernetes kubernetes]# cat /etc/kubernetes/kubeconfigbak
apiVersion: v1
kind: Config
users:
- name: kubelet
user:
client-certificate: /var/lib/kubernetes/kubelet_client.crt
client-key: /var/lib/kubernetes/kubelet_client.key
clusters:
- name: local
cluster:
certificate-authority: /var/lib/kubernetes/ca.crt
server: http://192.168.73.100:6443
contexts:
- context:
cluster: local
user: kubelet
name: my-context
current-context: my-context
设置kubelet启动参数
--kubeconfig=/etc/kubernetes/kubeconfigbak
[root@kubernetes kubernetes]# vi /etc/kubernetes/kubelet
[root@kubernetes kubernetes]#
[root@kubernetes kubernetes]#
[root@kubernetes kubernetes]# cat /etc/kubernetes/kubelet
KUBELET_ARGS="--kubeconfig=/etc/kubernetes/kubeconfigbak \
--hostname-override=192.168.73.100 \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=0"
[root@kubernetes kubernetes]# systemctl restart kubelet
[root@kubernetes kubernetes]# systemctl status kubelet
设置kube-proxy启动参数
--kubeconfig=/etc/kubernetes/kubeconfigbak
[root@kubernetes kubernetes]# vi /etc/kubernetes/proxy
[root@kubernetes kubernetes]# cat /etc/kubernetes/proxy
KUBE_PROXY_ARGS="--kubeconfig=/etc/kubernetes/kubeconfigbak \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2"
[root@kubernetes kubernetes]# systemctl restart kube-proxy
[root@kubernetes kubernetes]# systemctl status kube-proxy
1.4.2.node1
[root@kubernetes-node1 ~]# cd /var/lib/kubernetes/
[root@kubernetes-node1 kubernetes]# ls
ca.crt ca.key
[root@kubernetes-node1 kubernetes]# openssl genrsa -out kubelet_client.key 2048
Generating RSA private key, 2048 bit long modulus
................................+++
..................+++
e is 65537 (0x10001)
[root@kubernetes-node1 kubernetes]# openssl req -new -key kubelet_client.key -subj "/CN=192.168.73.101" -out kubelet_client.csr
[root@kubernetes-node1 kubernetes]# openssl x509 -req -in kubelet_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubelet_client.crt -days 5000
Signature ok
subject=/CN=192.168.73.101
Getting CA Private Key
[root@kubernetes-node1 kubernetes]# cp /etc/kubernetes/kubeconfig /etc/kubernetes/kubeconfig.bak
[root@kubernetes-node1 kubernetes]# vi /etc/kubernetes/kubeconfig
[root@kubernetes-node1 kubernetes]# cat /etc/kubernetes/kubeconfig
apiVersion: v1
kind: Config
users:
- name: kubelet
user:
client-certificate: /var/lib/kubernetes/kubelet_client.crt
client-key: /var/lib/kubernetes/kubelet_client.key
clusters:
- name: local
cluster:
certificate-authority: /var/lib/kubernetes/ca.crt
server: http://192.168.73.100:6443
contexts:
- context:
cluster: local
user: kubelet
name: my-context
current-context: my-context
设置kubelet启动参数
--kubeconfig=/etc/kubernetes/kubeconfig
原kubelet配置文件有此参数,直接重启kubelet服务
[root@kubernetes-node1 kubernetes]# systemctl restart kubelet
[root@kubernetes-node1 kubernetes]# systemctl status kubelet
设置kube-proxy启动参数
--kubeconfig=/etc/kubernetes/kubeconfig
原proxy配置文件有此参数,直接重启proxy服务
[root@kubernetes-node1 kubernetes]# systemctl restart kube-proxy
[root@kubernetes-node1 kubernetes]# systemctl status kube-proxy
1.4.3.node2
[root@kubernetes-node2 ~]# cd /var/lib/kubernetes/
[root@kubernetes-node2 kubernetes]# openssl genrsa -out kubelet_client.key 2048
Generating RSA private key, 2048 bit long modulus
...+++
...................................+++
e is 65537 (0x10001)
[root@kubernetes-node2 kubernetes]# openssl req -new -key kubelet_client.key -subj "/CN=192.168.73.102" -out kubelet_client.csr
[root@kubernetes-node2 kubernetes]# openssl x509 -req -in kubelet_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubelet_client.crt -days 5000
Signature ok
subject=/CN=192.168.73.102
Getting CA Private Key
[root@kubernetes-node2 kubernetes]# cp /etc/kubernetes/kubeconfig /etc/kubernetes/kubeconfig.bak
[root@kubernetes-node2 kubernetes]# vi /etc/kubernetes/kubeconfig
[root@kubernetes-node2 kubernetes]#
[root@kubernetes-node2 kubernetes]#
[root@kubernetes-node2 kubernetes]# cat /etc/kubernetes/kubeconfig
apiVersion: v1
kind: Config
users:
- name: kubelet
user:
client-certificate: /var/lib/kubernetes/kubelet_client.crt
client-key: /var/lib/kubernetes/kubelet_client.key
clusters:
- name: local
cluster:
certificate-authority: /var/lib/kubernetes/ca.crt
server: http://192.168.73.100:6443
contexts:
- context:
cluster: local
user: kubelet
name: my-context
current-context: my-context
[root@kubernetes-node2 kubernetes]#
[root@kubernetes-node2 kubernetes]# systemctl restart kubelet
[root@kubernetes-node2 kubernetes]# systemctl status kubelet
[root@kubernetes-node2 kubernetes]# systemctl restart kube-proxy
[root@kubernetes-node2 kubernetes]# systemctl status kube-proxy
1.5.设置kubectl客户端使用安全访问API Server
安全访问需要带上安全参数,如下
--certificate-authority :使用为kube-apiserver生成的ca.crt文件
--client-certificate : 使用为kube-controller-manager生成的cs_client.crt文件
--client-key :使用为kube-controller-manager生成的cs_client.key文件
示例:
kubectl --server=https://192.168.73.100:6443 \
--certificate-authority=/var/lib/kubernetes/ca.crt \
--client-certificate=/var/lib/kubernetes/cs_client.crt \
--client-key=/var/lib/kubernetes/cs_client.key \
get nodes
[root@kubernetes kubernetes]# kubectl --server=https://192.168.73.100:6443 --certificate-authority=/var/lib/kubernetes/ca.crt --client-certificate=/var/lib/kubernetes/cs_client.crt --client-key=/var/lib/kubernetes/cs_client.key get nodes
NAME STATUS ROLES AGE VERSION
192.168.73.100 Ready <none> 22h v1.17.3
192.168.73.101 Ready <none> 22h v1.17.3
192.168.73.102 Ready <none> 21h v1.17.3
2.0基于HTTP Base认证配置过程
创建用户名,密码,UID文件
[root@kubernetes kubernetes]# cd /etc/kubernetes/
[root@kubernetes kubernetes]# vim basic_auth_file
[root@kubernetes kubernetes]# cat basic_auth_file
admin,admin,1
system,system,2
设置apiserver启动参数
--basic-auth-file=/etc/kubernetes/basic_auth_file
[root@kubernetes kubernetes]# vi /etc/kubernetes/apiserver
[root@kubernetes kubernetes]# cat /etc/kubernetes/apiserver
KUBE_API_ARGS=" \
--etcd-servers=http://127.0.0.1:2379 \
--insecure-bind-address=0.0.0.0 \
--insecure-port=0 \
--service-cluster-ip-range=169.169.0.0/16 \
--service-node-port-range=1-65535 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,DefaultStorageClass,DefaultTolerationSeconds,ValidatingAdmissionWebhook,ResourceQuota \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=0 \
--client-ca-file=/var/lib/kubernetes/ca.crt \
--tls-private-key-file=/var/lib/kubernetes/server.key \
--tls-cert-file=/var/lib/kubernetes/server.crt \
--secure-port=6443 \
--basic-auth-file=/etc/kubernetes/basic_auth_file"
[root@kubernetes kubernetes]# systemctl restart kube-apiserver
[root@kubernetes kubernetes]# systemctl status kube-apiserver
使用kubectl通过指定的用户名及密码访问API Server
[root@kubernetes kubernetes]# [root@kubernetes kubernetes]# kubectl --server=https://192.168.73.100:6443 --username=admin --password=admin --insecure-skip-tls-verify=true get nodes
NAME STATUS ROLES AGE VERSION
192.168.73.100 Ready <none> 22h v1.17.3
192.168.73.101 Ready <none> 22h v1.17.3
192.168.73.102 Ready <none> 21h v1.17.3
3.0基于Token认证的配置
创建用户名,密码,UID文件
[root@kubernetes kubernetes]# cd /etc/kubernetes/
[root@kubernetes kubernetes]# vim token_auth_file
[root@kubernetes kubernetes]# cat token_auth_file
admin,admin,1
system,system,2
设置apiserver启动参数
--token-auth-file=/etc/kubernetes/token_auth_file
[root@kubernetes kubernetes]# vi /etc/kubernetes/apiserver
[root@kubernetes kubernetes]# cat /etc/kubernetes/apiserver
KUBE_API_ARGS=" \
--etcd-servers=http://127.0.0.1:2379 \
--insecure-bind-address=0.0.0.0 \
--insecure-port=0 \
--service-cluster-ip-range=169.169.0.0/16 \
--service-node-port-range=1-65535 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,DefaultStorageClass,DefaultTolerationSeconds,ValidatingAdmissionWebhook,ResourceQuota \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=0 \
--client-ca-file=/var/lib/kubernetes/ca.crt \
--tls-private-key-file=/var/lib/kubernetes/server.key \
--tls-cert-file=/var/lib/kubernetes/server.crt \
--secure-port=6443 \
--basic-auth-file=/etc/kubernetes/basic_auth_file \
--token-auth-file=/etc/kubernetes/token_auth_file"
[root@kubernetes kubernetes]# systemctl restart kube-apiserver
[root@kubernetes kubernetes]# systemctl status kube-apiserver
使用curl访问API Server
[root@kubernetes kubernetes]# curl -k --header "Authorization:Bearer admin" https://192.168.73.100:6443/version
{
"major": "1",
"minor": "17",
"gitVersion": "v1.17.3",
"gitCommit": "06ad960bfd03b39c8310aaf92d1e7c12ce618213",
"gitTreeState": "clean",
"buildDate": "2020-02-11T18:07:13Z",
"goVersion": "go1.13.6",
"compiler": "gc",
"platform": "linux/amd64"
}
4.设置别名,进行快捷操作
[root@kubernetes kubernetes]# kubectl get namespaces
The connection to the server localhost:8080 was refused - did you specify the right host or port?
[root@kubernetes ~]# vim /root/.bashrc
[root@kubernetes ~]# cat /root/.bashrc
# .bashrc
# User specific aliases and functions
alias rm='rm -i'
alias cp='cp -i'
alias mv='mv -i'
# Source global definitions
if [ -f /etc/bashrc ]; then
. /etc/bashrc
fi
alias kubectl='kubectl --server=https://192.168.73.100:6443 --username=admin --password=admin --insecure-skip-tls-verify=true'
[root@kubernetes ~]#
[root@kubernetes ~]# source /root/.bashrc
[root@kubernetes ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
192.168.73.100 Ready <none> 22h v1.17.3
192.168.73.101 Ready <none> 22h v1.17.3
192.168.73.102 Ready <none> 22h v1.17.3