HDFS管理与优化-优快云博客

HDFS文件统计

hdfs dfs -du /user/hive/warehouse/ods.db/ > db_du

hdfs dfs -ls /user/hive/warehouse/udm.db/ > db_ls

合并信息脚本

#!/usr/bin/perl
use strict;
my %table_du = map{my @m = split /\s+/;$m[0],$m[1]} `awk -F'[ /]' '{print tolower(\$NF),\$1}' db_du`;
my %table_ls = map{my @m = split /\s+/;$m[0],$m[1]} `awk '{print tolower(\$NF),\$6}' db_ls|awk -F\/ '{print \$NF}'`;

for (keys %table_du){
    print "$_|$table_du{$_}|$table_ls{$_}\n";
}

HDFS 过期文件清理

#shell脚本
#!/bin/bash
#cleanup hive session dir on hdfs named with Specify the directory
usage="Usage: hive-clean.sh [days]"
cleanup_dir="/tmp/hive/bduser"
echo "cleanup_dir on HDFS:  $cleanup_dir"
if [ ! "$1" ]; then
    echo $usage
    exit 1
fi
now=$(date +%s)
hadoop fs -ls $cleanup_dir | grep "^d" | while read f; do
    dir_date=$(echo $f | awk '{print $6}')
    difference=$((($now - $(date -d "$dir_date" +%s)) / (24 * 60 * 60)))
    if [ $difference -gt $1 ]; then
        echo $f
        # 此处直接对上边的过滤结果进行删除，建议先改成 hadoop fs -ls -R 命令先进行测试，一切正常在改回删除
        hadoop fs -ls -R $(echo $f | awk '{ print $8 }')
        #hadoop fs -rm -r -skipTrash $(echo $f | awk '{ print $8 }')
    fi
done