HDFS删除过期数据

最新推荐文章于 2025-05-12 08:24:49 发布

sunxunyong

最新推荐文章于 2025-05-12 08:24:49 发布

阅读量104

点赞数 2

文章标签： hdfs hadoop 大数据

本文链接：https://blog.youkuaiyun.com/sunxunyong/article/details/145493258

版权

#!/bin/bash
#set -x

source ~/.bashrc
# HADOOP所在的bin目录
HADOOP_BIN_PATH=/opt/cloudera/parcels/CDH/bin

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

echo $bin
if [ $# -lt 1 ]
then
  echo "Usage: ./delshell.sh  MachineTag" 
  echo "Usage: ./delshell.sh  MachineTag confFile" 
  exit
fi

tag=$1
if [ 'a'$2'a' == 'aa' ]
then
  confFile="$bin"/ddds.conf
else
  confFile=$
fi

#/tmp@30,del1,

if [ -f $confFile ]
then
    for ddd in `cat $confFile|grep -v '^#'|grep ','$tag','|awk -F',' '{print $1}'`
    do
       echo "*******************$ddd***************************"   
           array=(${ddd//@/ }) 
           echo "${array[0]}"
           echo "${array[1]}" 
           $HADOOP_BIN_PATH/hdfs dfs -ls "${array[0]}" |grep -E "^d"  |awk '{gsub("-", "", $6)}; {print $6" "$8}'|awk '$1<(strftime("%Y%m%d")-"${array[1]}") {print $2}' | xargs hdfs dfs -ls  
    done
else
  echo "Error: Please assign config file or run delshell.sh command with ddds.conf in same directory" 
fi