K8S 创建PVC 时提示如下的报错:
[root@k8s1 k8s-ceph]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
storageclass-pvc01 Pending storageclass01 12h
[root@k8s1 k8s-ceph]# kubectl describe pvc Pending
Error from server (NotFound): persistentvolumeclaims "Pending" not found
[root@k8s1 k8s-ceph]# kubectl describe pvc storageclass-pvc01
Name: storageclass-pvc01
Namespace: default
StorageClass: storageclass01
Status: Pending
Volume:
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration:
{"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"name":"storageclass-pvc01","namespace":"default"},"spec":{...
volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/rbd
Finalizers: [kubernetes.io/pvc-protection]
Capacity:
Access Modes:
VolumeMode: Filesystem
Mounted By: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning ProvisioningFailed 86s (x350 over 12h) persistentvolume-controller Failed to provision volume with StorageClass "storageclass01": failed to create rbd image: executable file not found in $PATH, command output:
[root@k8s1 k8s-ceph]#
解决过程:
Last login: Fri Nov 15 10:42:11 2019 from 10.83.83.78
[root@k8s1 ~]#
[root@k8s1 ~]#
[root@k8s1 ~]#
[root@k8s1 ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
storageclass-pvc01 Pending storageclass01 13h
[root@k8s1 ~]# kubectl delete pvc storageclass-pvc01
persistentvolumeclaim "storageclass-pvc01" deleted
[root@k8s1 ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
[root@k8s1 ~]# kubectl get storageclass
NAME PROVISIONER AGE
storageclass01 kubernetes.io/rbd 13h
[root@k8s1 ~]# kubectl delete storageclass storageclass01
storageclass.storage.k8s.io "storageclass01" deleted
[root@k8s1 ~]# kubectl get storageclass
No resources found in default namespace.
[root@k8s1 ~]# kubectl get storageclass
No resources found in default namespace.
[root@k8s1 ~]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret-admin kubernetes.io/rbd 1 42h
ceph-secret-rbd kubernetes.io/rbd 1 25h
default-token-c9rjq kubernetes.io/service-account-token 3 8d
kubernetes-dashboard-certs Opaque 2 7d19h
prometheus-operator-token-hzhb4 kubernetes.io/service-account-token 3 40h
prometheus-token-4dnn6 kubernetes.io/service-account-token 3 40h
[root@k8s1 ~]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret-admin kubernetes.io/rbd 1 42h
ceph-secret-rbd kubernetes.io/rbd 1 25h
default-token-c9rjq kubernetes.io/service-account-token 3 8d
kubernetes-dashboard-certs Opaque 2 7d19h
prometheus-operator-token-hzhb4 kubernetes.io/service-account-token 3 40h
prometheus-token-4dnn6 kubernetes.io/service-account-token 3 40h
[root@k8s1 ~]#
[root@k8s1 ~]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret-admin kubernetes.io/rbd 1 42h
ceph-secret-rbd kubernetes.io/rbd 1 25h
default-token-c9rjq kubernetes.io/service-account-token 3 8d
kubernetes-dashboard-certs Opaque 2 7d19h
prometheus-operator-token-hzhb4 kubernetes.io/service-account-token 3 40h
prometheus-token-4dnn6 kubernetes.io/service-account-token 3 40h
[root@k8s1 ~]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret-admin kubernetes.io/rbd 1 42h
ceph-secret-rbd kubernetes.io/rbd 1 25h
default-token-c9rjq kubernetes.io/service-account-token 3 8d
kubernetes-dashboard-certs Opaque 2 7d19h
prometheus-operator-token-hzhb4 kubernetes.io/service-account-token 3 40h
prometheus-token-4dnn6 kubernetes.io/service-account-token 3 40h
[root@k8s1 ~]# ls
anaconda-ks.cfg dashboard.crt dashboard.key k8s-ceph kube-flannel.yml kubernetes-dashboard.yaml.1.10 yml
calico.yaml dashboard.csr google kubeadm.yaml kubernetes-dashboard.yaml recommended.yaml
[root@k8s1 ~]# cd k8s-ceph/
[root@k8s1 k8s-ceph]# ls
ceph-rbd-pod-4.yaml ceph-sc-rbd-2.yaml ceph-secret.yaml pv.yaml rbdpd.yaml storageclass-pvc.yaml
ceph-rbd-pvc-3.yaml ceph-secret-rbd-1.yaml pvc.yaml rbdpd_new.yaml storageclass-pvc-pod.yaml storageclass.yaml
[root@k8s1 k8s-ceph]# cat storageclass.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: storageclass01
provisioner: kubernetes.io/rbd
parameters:
monitors: 10.83.35.79:6789,10.83.35.80:6789,10.83.35.81:6789
adminId: admin
adminSecretName: ceph-secret-rbd
adminSecretNamespace: default
pool: kubernets
userId: admin
userSecretName: ceph-secret-rbd
fsType: ext4
imageFormat: "2"
imageFeatures: "layering"
[root@k8s1 k8s-ceph]# kubectl apply -f storageclass.yaml
storageclass.storage.k8s.io/storageclass01 created
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# kubectl get storageclass
NAME PROVISIONER AGE
storageclass01 kubernetes.io/rbd 10s
[root@k8s1 k8s-ceph]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s1 Ready master 8d v1.16.2
k8s2 NotReady <none> 8d v1.16.0
k8s3 Ready <none> 8d v1.16.2
[root@k8s1 k8s-ceph]# kubectl get service
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
example-app ClusterIP 10.103.107.104 <none> 8080/TCP 3d1h
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 40h
[root@k8s1 k8s-ceph]# kubectl get pod
NAME READY STATUS RESTARTS AGE
bqjr01-5cc79db9bf-jqxjl 1/1 Running 2 2d17h
bqjr02-57659797f5-rzpt2 1/1 Running 2 2d17h
httpd-app-c77bb8b47-6q549 0/1 ContainerCreating 0 14s
httpd-app-c77bb8b47-bnf5t 1/1 Terminating 1 2d17h
httpd-app-c77bb8b47-hfjwj 1/1 Running 2 2d17h
nginx-86c57db685-d25gf 1/1 Terminating 1 2d17h
nginx-86c57db685-n8csl 0/1 ContainerCreating 0 14s
nginx-86c57db685-tw9x5 1/1 Running 2 2d17h
nginx-86c57db685-w8pq9 0/1 ContainerCreating 0 14s
nginx-86c57db685-zjc9h 1/1 Terminating 1 2d17h
prometheus-operator-99dccdc56-92h97 0/1 Terminating 308 3d16h
prometheus-operator-99dccdc56-wwjt8 0/1 ContainerCreating 0 14s
rbd000new 1/1 Terminating 0 19h
[root@k8s1 k8s-ceph]# kubectl get service
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
example-app ClusterIP 10.103.107.104 <none> 8080/TCP 3d1h
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 40h
[root@k8s1 k8s-ceph]# kubectl get storageclass
NAME PROVISIONER AGE
storageclass01 kubernetes.io/rbd 80s
[root@k8s1 k8s-ceph]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
[root@k8s1 k8s-ceph]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pv01 1Gi RWO Recycle Bound default/pvc01 18h
[root@k8s1 k8s-ceph]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
[root@k8s1 k8s-ceph]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
[root@k8s1 k8s-ceph]# ls
ceph-rbd-pod-4.yaml ceph-sc-rbd-2.yaml ceph-secret.yaml pv.yaml rbdpd.yaml storageclass-pvc.yaml
ceph-rbd-pvc-3.yaml ceph-secret-rbd-1.yaml pvc.yaml rbdpd_new.yaml storageclass-pvc-pod.yaml storageclass.yaml
[root@k8s1 k8s-ceph]# kubectl apply -f storageclass-pvc.yaml
persistentvolumeclaim/storageclass-pvc01 created
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
storageclass-pvc01 Pending storageclass01 5s
[root@k8s1 k8s-ceph]# kubectl describe storageclass-pvc01
error: the server doesn't have a resource type "storageclass-pvc01"
[root@k8s1 k8s-ceph]# kubectl describe pvc storageclass-pvc01
Name: storageclass-pvc01
Namespace: default
StorageClass: storageclass01
Status: Pending
Volume:
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration:
{"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"name":"storageclass-pvc01","namespace":"default"},"spec":{...
volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/rbd
Finalizers: [kubernetes.io/pvc-protection]
Capacity:
Access Modes:
VolumeMode: Filesystem
Mounted By: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning ProvisioningFailed 5s (x3 over 33s) persistentvolume-controller Failed to provision volume with StorageClass "storageclass01": failed to create rbd image: executable file not found in $PATH, command output:
[root@k8s1 k8s-ceph]# yum install -y ceph-common
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
* base: mirrors.aliyun.com
* extras: mirrors.aliyun.com
* updates: mirrors.aliyun.com
Ceph | 2.9 kB 00:00:00
Ceph-noarch | 2.9 kB 00:00:00
base | 3.6 kB 00:00:00
ceph-source | 2.9 kB 00:00:00
docker-ce-stable | 3.5 kB 00:00:00
epel | 5.4 kB 00:00:00
extras | 2.9 kB 00:00:00
kubernetes/signature | 454 B 00:00:00
kubernetes/signature | 1.4 kB 00:00:01 !!!
updates | 2.9 kB 00:00:00
(1/2): kubernetes/primary | 59 kB 00:00:00
(2/2): docker-ce-stable/x86_64/primary_db | 37 kB 00:00:00
kubernetes 430/430
Package 2:ceph-common-14.2.4-0.el7.x86_64 already installed and latest version
Nothing to do
[root@k8s1 k8s-ceph]# kubectl describe pvc storageclass-pvc01
Name: storageclass-pvc01
Namespace: default
StorageClass: storageclass01
Status: Pending
Volume:
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration:
{"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"name":"storageclass-pvc01","namespace":"default"},"spec":{...
volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/rbd
Finalizers: [kubernetes.io/pvc-protection]
Capacity:
Access Modes:
VolumeMode: Filesystem
Mounted By: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning ProvisioningFailed 9s (x5 over 67s) persistentvolume-controller Failed to provision volume with StorageClass "storageclass01": failed to create rbd image: executable file not found in $PATH, command output:
[root@k8s1 k8s-ceph]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s1 Ready master 8d v1.16.2
k8s2 NotReady <none> 8d v1.16.0
k8s3 Ready <none> 8d v1.16.2
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s1 Ready master 8d v1.16.2
k8s2 Ready <none> 8d v1.16.2
k8s3 Ready <none> 8d v1.16.2
[root@k8s1 k8s-ceph]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s1 Ready master 8d v1.16.2
k8s2 Ready <none> 8d v1.16.2
k8s3 Ready <none> 8d v1.16.2
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s1 Ready master 8d v1.16.2
k8s2 Ready <none> 8d v1.16.2
k8s3 Ready <none> 8d v1.16.2
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
storageclass-pvc01 Pending storageclass01 8m56s
[root@k8s1 k8s-ceph]# kubectl delete pvc storageclass-pvc01
persistentvolumeclaim "storageclass-pvc01" deleted
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# kubectl delete sc storageclass01
storageclass.storage.k8s.io "storageclass01" deleted
[root@k8s1 k8s-ceph]# kubectl get storageclass
No resources found in default namespace.
[root@k8s1 k8s-ceph]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret-admin kubernetes.io/rbd 1 42h
ceph-secret-rbd kubernetes.io/rbd 1 25h
default-token-c9rjq kubernetes.io/service-account-token 3 8d
kubernetes-dashboard-certs Opaque 2 7d19h
prometheus-operator-token-hzhb4 kubernetes.io/service-account-token 3 40h
prometheus-token-4dnn6 kubernetes.io/service-account-token 3 40h
[root@k8s1 k8s-ceph]# ls
ceph-rbd-pod-4.yaml ceph-sc-rbd-2.yaml ceph-secret.yaml pv.yaml rbdpd.yaml storageclass-pvc.yaml
ceph-rbd-pvc-3.yaml ceph-secret-rbd-1.yaml pvc.yaml rbdpd_new.yaml storageclass-pvc-pod.yaml storageclass.yaml
[root@k8s1 k8s-ceph]# kubectl apply -f storageclass.yaml
storageclass.storage.k8s.io/storageclass01 created
[root@k8s1 k8s-ceph]# kubectl apply -f storageclass-pvc.yaml
persistentvolumeclaim/storageclass-pvc01 created
[root@k8s1 k8s-ceph]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc01 Bound pv01 1Gi RWO 18h
storageclass-pvc01 Pending storageclass01 16s
[root@k8s1 k8s-ceph]# kubectl describe pvc storageclass-pvc01
Name: storageclass-pvc01
Namespace: default
StorageClass: storageclass01
Status: Pending
Volume:
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration:
{"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"name":"storageclass-pvc01","namespace":"default"},"spec":{...
volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/rbd
Finalizers: [kubernetes.io/pvc-protection]
Capacity:
Access Modes:
VolumeMode: Filesystem
Mounted By: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning ProvisioningFailed 9s (x4 over 41s) persistentvolume-controller Failed to provision volume with StorageClass "storageclass01": failed to create rbd image: executable file not found in $PATH, command output:
[root@k8s1 k8s-ceph]# ls
ceph-rbd-pod-4.yaml ceph-sc-rbd-2.yaml ceph-secret.yaml pv.yaml rbdpd.yaml storageclass-pvc.yaml
ceph-rbd-pvc-3.yaml ceph-secret-rbd-1.yaml pvc.yaml rbdpd_new.yaml storageclass-pvc-pod.yaml storageclass.yaml
[root@k8s1 k8s-ceph]# cat storageclass.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: storageclass01
provisioner: kubernetes.io/rbd
parameters:
monitors: 10.83.35.79:6789,10.83.35.80:6789,10.83.35.81:6789
adminId: admin
adminSecretName: ceph-secret-rbd
adminSecretNamespace: default
pool: kubernets
userId: admin
userSecretName: ceph-secret-rbd
fsType: ext4
imageFormat: "2"
imageFeatures: "layering"
[root@k8s1 k8s-ceph]# ls
ceph-rbd-pod-4.yaml ceph-sc-rbd-2.yaml ceph-secret.yaml pv.yaml rbdpd.yaml storageclass-pvc.yaml
ceph-rbd-pvc-3.yaml ceph-secret-rbd-1.yaml pvc.yaml rbdpd_new.yaml storageclass-pvc-pod.yaml storageclass.yaml
[root@k8s1 k8s-ceph]# cat ceph-secret-rbd-1.yaml
apiVersion: v1
kind: Secret
metadata:
name: ceph-secret-rbd
type: "kubernetes.io/rbd"
data:
key: QVFBZkxNRmRwdE9kTVJBQTI0enFGYng1a2FLd0xwdXJvUFYwVWc9PQo=
[root@k8s1 k8s-ceph]# cat /etc/ceph/ceph.client.admin.keyring
[client.admin]
key = AQAfLMFdptOdMRAA24zqFbx5kaKwLpuroPV0Ug==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
[root@k8s1 k8s-ceph]# echo "AQAfLMFdptOdMRAA24zqFbx5kaKwLpuroPV0Ug=="|base64
QVFBZkxNRmRwdE9kTVJBQTI0enFGYng1a2FLd0xwdXJvUFYwVWc9PQo=
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# cat /etc/ceph/ceph.client.admin.keyring
[client.admin]
key = AQAfLMFdptOdMRAA24zqFbx5kaKwLpuroPV0Ug==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# vim secret01.yml
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
"secret01.yml" [New] 8L, 136C written
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# cat secret01.yml
apiVersion: v1
kind: Secret
metadata:
name: secret01
type: "kubernetes.io/rbd"
data:
key: AQAfLMFdptOdMRAA24zqFbx5kaKwLpuroPV0Ug==
[root@k8s1 k8s-ceph]# kubectl apply -f secret01.yml
secret/secret01 created
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]#
[root@k8s1 k8s-ceph]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret-admin kubernetes.io/rbd 1 43h
ceph-secret-rbd kubernetes.io/rbd 1 25h
default-token-c9rjq kubernetes.io/service-account-token 3 8d
kubernetes-dashboard-certs Opaque 2 7d20h
prometheus-operator-token-hzhb4 kubernetes.io/service-account-token 3 41h
prometheus-token-4dnn6 kubernetes.io/service-account-token 3 41h
secret01 kubernetes.io/rbd 1 10s
[root@k8s1 k8s-ceph]# kubectl get secret -n default
NAME TYPE DATA AGE
ceph-secret-admin kubernetes.io/rbd 1 43h
ceph-secret-rbd kubernetes.io/