官方参考文档：
GitHub - ceph/ceph-csi at v3.9.0
测试版本
| Ceph Version | Ceph CSI Version | Container Orchestrator Name | Version Tested |
| v17.2.7 | v3.9.0 | Kubernetes | v1.25.6 |
安装Ceph-csi
Step 1 Download GitHub - ceph/ceph-csi at v3.9.0
root@sd-k8s-master-1:~# wget https://github.com/ceph/ceph-csi/archive/refs/tags/v3.9.0.zip
root@sd-k8s-master-1:~# unzip v3.9.0.zip
Step 2 创建csidriver
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csidriver.yaml
#
# /!\ DO NOT MODIFY THIS FILE
#
# This file has been automatically generated by Ceph-CSI yamlgen.
# The source for the contents can be found in the api/deploy directory, make
# your modifications there.
#
---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: "cephfs.csi.ceph.com"
namespace: cephfs
spec:
attachRequired: false
podInfoOnMount: false
fsGroupPolicy: File
seLinuxMount: true
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csidriver.yaml
Step 3 为sidecar容器和节点插件部署RBAC：
root@sd-k8s-master-1:~# cd /root/ceph-csi-3.9.0/deploy/cephfs/kubernetes
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-provisioner-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cephfs-csi-provisioner
namespace: cephfs
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["get", "list", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get"]
- apiGroups: [""]
resources: ["serviceaccounts/token"]
verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: cephfs-csi-provisioner
namespace: cephfs
roleRef:
kind: ClusterRole
name: cephfs-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
# replace with non-cephfs namespace name
namespace: cephfs
name: cephfs-external-provisioner-cfg
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role-cfg
# replace with non-cephfs namespace name
namespace: cephfs
subjects:
- kind: ServiceAccount
name: cephfs-csi-provisioner
# replace with non-cephfs namespace name
namespace: cephfs
roleRef:
kind: Role
name: cephfs-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-nodeplugin-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cephfs-csi-nodeplugin
namespace: cephfs
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get"]
- apiGroups: [""]
resources: ["serviceaccounts/token"]
verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: cephfs-csi-nodeplugin
# replace with non-cephfs namespace name
namespace: cephfs
roleRef:
kind: ClusterRole
name: cephfs-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-provisioner-rbac.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-nodeplugin-rbac.yaml
Step 4 编辑configmap文件
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-config-map.yaml
apiVersion: v1
kind: ConfigMap
data:
config.json: |-
[
{
"clusterID": "92ab6c78-7edc-11ee-aec4-5e807f521aec",
"monitors": [
"10.220.9.13:6789",
"10.220.9.14:6789",
"10.220.9.15:6789"
]
}
]
metadata:
name: ceph-csi-config
namespace: cephfs
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-config.yaml
apiVersion: v1
kind: ConfigMap
data:
ceph.conf: |
[global]
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
# adding 'fuse_big_writes = true' option by default to override this limit
# see https://github.com/ceph/ceph-csi/issues/1928
fuse_big_writes = true
# keyring is a required key and its value should be empty
keyring: |
metadata:
name: ceph-config
namespace: cephfs
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-csi-config.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-config.yaml
Step 5 部署 CSI Sidecar 容器：
#注释ceph-csi-encryption-kms-config volume
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# sed -i 's/registry.k8s.io/k8s.dockerproxy.com/g' csi-cephfsplugin-provisioner.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# sed -i 's/registry.k8s.io/k8s.dockerproxy.com/g' csi-cephfsplugin.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-cephfsplugin-provisioner.yaml
---
kind: Service
apiVersion: v1
metadata:
name: csi-cephfsplugin-provisioner
namespace: cephfs
labels:
app: csi-metrics
spec:
selector:
app: csi-cephfsplugin-provisioner
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8681
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: csi-cephfsplugin-provisioner
namespace: cephfs
spec:
selector:
matchLabels:
app: csi-cephfsplugin-provisioner
replicas: 3
template:
metadata:
labels:
app: csi-cephfsplugin-provisioner
spec:
# affinity:
# podAntiAffinity:
# requiredDuringSchedulingIgnoredDuringExecution:
# - labelSelector:
# matchExpressions:
# - key: app
# operator: In
# values:
# - csi-cephfsplugin-provisioner
# topologyKey: "kubernetes.io/hostname"
serviceAccountName: cephfs-csi-provisioner
priorityClassName: system-cluster-critical
containers:
- name: csi-provisioner
image: k8s.dockerproxy.com/sig-storage/csi-provisioner:v3.5.0
args:
- "--csi-address=$(ADDRESS)"
- "--v=1"
- "--timeout=150s"
- "--leader-election=true"
- "--retry-interval-start=500ms"
- "--feature-gates=Topology=false"
- "--feature-gates=HonorPVReclaimPolicy=true"
- "--prevent-volume-mode-conversion=true"
- "--extra-create-metadata=true"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-resizer
image: k8s.dockerproxy.com/sig-storage/csi-resizer:v1.8.0
args:
- "--csi-address=$(ADDRESS)"
- "--v=1"
- "--timeout=150s"
- "--leader-election"
- "--retry-interval-start=500ms"
- "--handle-volume-inuse-error=false"
- "--feature-gates=RecoverVolumeExpansionFailure=true"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-snapshotter
image: k8s.dockerproxy.com/sig-storage/csi-snapshotter:v6.2.2
args:
- "--csi-address=$(ADDRESS)"
- "--v=1"
- "--timeout=150s"
- "--leader-election=true"
- "--extra-create-metadata=true"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-cephfsplugin
image: quay.io/cephcsi/cephcsi:v3.9.0
args:
- "--nodeid=$(NODE_ID)"
- "--type=cephfs"
- "--controllerserver=true"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=cephfs.csi.ceph.com"
- "--pidlimit=-1"
- "--enableprofiling=false"
- "--setmetadata=true"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
# - name: KMS_CONFIGMAP_NAME
# value: encryptionConfig
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: host-sys
mountPath: /sys
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
- name: ceph-config
mountPath: /etc/ceph/
- name: ceph-csi-config
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
# - name: ceph-csi-encryption-kms-config
## # mountPath: /etc/ceph-csi-encryption-kms-config/
- name: liveness-prometheus
image: quay.io/cephcsi/cephcsi:v3.9.0
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=8681"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: socket-dir
emptyDir: {
medium: "Memory"
}
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: host-dev
hostPath:
path: /dev
- name: ceph-config
configMap:
name: ceph-config
- name: ceph-csi-config
configMap:
name: ceph-csi-config
- name: keys-tmp-dir
emptyDir: {
medium: "Memory"
}
# - name: ceph-csi-encryption-kms-config
# configMap:
# name: ceph-csi-encryption-kms-config
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat csi-cephfsplugin.yaml
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-cephfsplugin
namespace: cephfs
spec:
selector:
matchLabels:
app: csi-cephfsplugin
template:
metadata:
labels:
app: csi-cephfsplugin
spec:
serviceAccountName: cephfs-csi-nodeplugin
priorityClassName: system-node-critical
hostNetwork: true
hostPID: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
# This is necessary only for systems with SELinux, where
# non-privileged sidecar containers cannot access unix domain socket
# created by privileged CSI driver container.
securityContext:
privileged: true
allowPrivilegeEscalation: true
image: k8s.dockerproxy.com/sig-storage/csi-node-driver-registrar:v2.8.0
args:
- "--v=1"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock"
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: quay.io/cephcsi/cephcsi:v3.9.0
args:
- "--nodeid=$(NODE_ID)"
- "--type=cephfs"
- "--nodeserver=true"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=cephfs.csi.ceph.com"
- "--enableprofiling=false"
# If topology based provisioning is desired, configure required
# node labels representing the nodes topology domain
# and pass the label names below, for CSI to consume and advertise
# its equivalent topology domain
# - "--domainlabels=failure-domain/region,failure-domain/zone"
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
# - name: KMS_CONFIGMAP_NAME
# value: encryptionConfig
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: mountpoint-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: Bidirectional
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins
mountPropagation: "Bidirectional"
- name: host-sys
mountPath: /sys
- name: etc-selinux
mountPath: /etc/selinux
readOnly: true
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
- name: host-mount
mountPath: /run/mount
- name: ceph-config
mountPath: /etc/ceph/
- name: ceph-csi-config
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
- name: ceph-csi-mountinfo
mountPath: /csi/mountinfo
# - name: ceph-csi-encryption-kms-config
# mountPath: /etc/ceph-csi-encryption-kms-config/
- name: liveness-prometheus
securityContext:
privileged: true
allowPrivilegeEscalation: true
image: quay.io/cephcsi/cephcsi:v3.9.0
args:
- "--type=liveness"
- "--endpoint=$(CSI_ENDPOINT)"
- "--metricsport=8681"
- "--metricspath=/metrics"
- "--polltime=60s"
- "--timeout=3s"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: "IfNotPresent"
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: mountpoint-dir
hostPath:
path: /var/lib/kubelet/pods
type: DirectoryOrCreate
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins
type: Directory
- name: host-sys
hostPath:
path: /sys
- name: etc-selinux
hostPath:
path: /etc/selinux
- name: lib-modules
hostPath:
path: /lib/modules
- name: host-dev
hostPath:
path: /dev
- name: host-mount
hostPath:
path: /run/mount
- name: ceph-config
configMap:
name: ceph-config
- name: ceph-csi-config
configMap:
name: ceph-csi-config
- name: keys-tmp-dir
emptyDir: {
medium: "Memory"
}
- name: ceph-csi-mountinfo
hostPath:
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/mountinfo
type: DirectoryOrCreate
#- name: ceph-csi-encryption-kms-config
# configMap:
# name: ceph-csi-encryption-kms-config
---
# This is a service to expose the liveness metrics
apiVersion: v1
kind: Service
metadata:
name: csi-metrics-cephfsplugin
namespace: cephfs
labels:
app: csi-metrics
spec:
ports:
- name: http-metrics
port: 8080
protocol: TCP
targetPort: 8681
selector:
app: csi-cephfsplugin
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-cephfsplugin-provisioner.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f csi-cephfsplugin.yaml
Step 6 Ceph创建cephfs
[root@ceph01 ~]# ceph osd pool cephfs
[root@ceph01 ~]# ceph auth get-key client.admin
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
Step 7 创建ceph secret
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-sc.yaml
apiVersion: v1
kind: Secret
metadata:
name: csi-cephfs-secret
namespace: cephfs
stringData:
userID: admin
userKey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
adminID: admin
adminKey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-sc.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get secrets -n cephfs
NAME TYPE DATA AGE
csi-cephfs-secret Opaque 4 49m
Step 8 创建storageclass
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat ceph-storage-class.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-cephfs-sc
provisioner: cephfs.csi.ceph.com
parameters:
clusterID: 92ab6c78-7edc-11ee-aec4-5e807f521aec
fsName: cephfs #cephfs名称，上面创建
pool: cephfs_data #cephfs pool名称
# mounter: fuse #挂载方式
csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
csi.storage.k8s.io/provisioner-secret-namespace: cephfs
csi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secret
csi.storage.k8s.io/controller-expand-secret-namespace: cephfs
csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
csi.storage.k8s.io/node-stage-secret-namespace: cephfs
reclaimPolicy: Delete
allowVolumeExpansion: true
#mountOptions:
# - discard
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f ceph-storage-class.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get storageclass
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
csi-cephfs-sc cephfs.csi.ceph.com Delete Immediate true 138m
Step 9 创建pvc测试
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat test-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cephfs-test-pvc
namespace: cephfs
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
storageClassName: csi-cephfs-sc
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f test-pvc.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get pvc -n cephfs
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
cephfs-test-pvc Bound pvc-94236c61-a5b7-494d-89ad-e1e18eaad175 1Gi RWX csi-cephfs-sc 49m
Step 10 创建pod测试
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# cat test-pod.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: test-pd
spec:
terminationGracePeriodSeconds: 0
containers:
- image: harbor.zetyun.cn/gcp/nginx:1.25.3
name: test-container
volumeMounts:
- mountPath: /cache
name: cache-volume
volumes:
- name: cache-volume
persistentVolumeClaim:
claimName: cephfs-test-pvc1 #填写pvc名称
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl create -f test-pod.yaml
root@sd-k8s-master-1:~/ceph-csi-3.9.0/deploy/cephfs/kubernetes# kubectl get po test-pd
NAME READY STATUS RESTARTS AGE
test-pd 1/1 Running 0 6m49s
扫一扫
1459
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
