前提
环境说明
1. 操作系统: centos 7.2
2. 应用: docker
现象:
1. / 磁盘不足
2. 实际上磁盘还有很多空间
3. 应用开始报错( out of space)
分析
利用 find 分析, 没有找到特殊的大文件
如:
find /apps -size +50M
利用 df 进行分析, 发现 / 已经用光
[root@hh-yun-k8s-128151 /]# df -ah
Filesystem Size Used Avail Use% Mounted on
rootfs - - - - /
sysfs 0 0 0 - /sys
proc 0 0 0 - /proc
devtmpfs 95G 0 95G 0% /dev
securityfs 0 0 0 - /sys/kernel/security
tmpfs 95G 0 95G 0% /dev/shm
devpts 0 0 0 - /dev/pts
tmpfs 95G 4.1G 91G 5% /run
tmpfs 95G 0 95G 0% /sys/fs/cgroup
cgroup 0 0 0 - /sys/fs/cgroup/systemd
pstore 0 0 0 - /sys/fs/pstore
cgroup 0 0 0 - /sys/fs/cgroup/cpuset
cgroup 0 0 0 - /sys/fs/cgroup/blkio
cgroup 0 0 0 - /sys/fs/cgroup/perf_event
cgroup 0 0 0 - /sys/fs/cgroup/cpu,cpuacct
cgroup 0 0 0 - /sys/fs/cgroup/hugetlb
cgroup 0 0 0 - /sys/fs/cgroup/devices
cgroup 0 0 0 - /sys/fs/cgroup/net_cls
cgroup 0 0 0 - /sys/fs/cgroup/freezer
cgroup 0 0 0 - /sys/fs/cgroup/memory
configfs 0 0 0 - /sys/kernel/config
/dev/sda2 80G 80G 286M 100% /
systemd-1 - - - - /proc/sys/fs/binfmt_misc
debugfs 0 0 0 - /sys/kernel/debug
mqueue 0 0 0 - /dev/mqueue
hugetlbfs 0 0 0 - /dev/hugepages
/dev/sda1 397M 135M 263M 34% /boot
tmpfs 19G 0 19G 0% /run/user/0
tmpfs 95G 12K 95G 1% /var/lib/kubelet/pods/fa7f705d-9a82-11e6-9bda-e09796534daa/volumes/kubernetes.io~secret/default-token-cuzo2
tmpfs 95G 12K 95G 1% /var/lib/kubelet/pods/32d9731a-bdfc-11e6-9bda-e09796534daa/volumes/kubernetes.io~secret/default-token-dqn5d
tmpfs 95G 12K 95G 1% /var/lib/kubelet/pods/c131bddd-c367-11e6-9bda-e09796534daa/volumes/kubernetes.io~secret/default-token-cuzo2
tmpfs 95G 12K 95G 1% /var/lib/kubelet/pods/27779000-c5cf-11e6-9bda-e09796534daa/volumes/kubernetes.io~secret/default-token-cuzo2
tmpfs 95G 12K 95G 1% /var/lib/kubelet/pods/277f9d4b-c5cf-11e6-9bda-e09796534daa/volumes/kubernetes.io~secret/default-token-cuzo2
tmpfs 95G 12K 95G 1% /var/lib/kubelet/pods/72505ec1-d570-11e6-b3fc-e09796534daa/volumes/kubernetes.io~secret/default-token-cuzo2
tmpfs 95G 12K 95G 1% /var/lib/kubelet/pods/b50b4c1c-d570-11e6-b3fc-e09796534daa/volumes/kubernetes.io~secret/default-token-cuzo2
binfmt_misc