一个完整Python 定时自动清理过期文件，并记录结果的完整程序

1，项目工程结构：

2，配置文件说明conf.ini

# encoding: utf-8

[sys]
#文件过期时间(单位天，可改为小时/分)
expire_day=7

#清理线程Sleep(单位秒)
sleep_interval=3600

#文件存储路径
clean_path=/tmp/data

#清理文件类型
clean_type={'.log', '.jpg', '.jpeg', '.bmp', '.png', '.gif'}

#日志级别
log_level=DEBUG

#日志存储位置
log_file=/var/log/file_clean.log

2，文件操作集合类fileutils.py：

#!/usr/bin/env python
# encoding: utf-8

##############################################################################
#   Function: 文件集合类                            #
#   Author:   why000007@163.com                                             #
#   Date:     09/12/2017                                                     #
##############################################################################


import os,sys
import os.path,time,datetime
import traceback
from file import RxrzError

def exists(fname):
    return True if os.path.exists(fname) else False

def isdir(fname):
    return True if os.path.isdir(fname) else False

def isfile(fname):
    return True if os.path.isfile(fname) else False

def isimage(fname):
    (_,ext) = os.path.splitext(fname)
    return True if ext=='.jpg' or ext=='.bmp' or ext=='.png' or ext=='.gif' or ext=='.feature' else False

def istype(fname, types):
    (_,ext) = os.path.splitext(fname)
    return True if ext in types else False

def isempty(fname):
    return True if exists(fname) and len(os.listdir(fname))==0 else False

def sctime(fname):
    return os.stat(fname).st_ctime

def smtime(fname):
    return os.stat(fname).st_mtime

def rmdirs(fname):
    try:
        if not exists(fname):
            return 0
        elif not isempty(fname):
            return -1
        else:
            os.removedirs(fname)
            return 1
    except OSError, oe:
        raise RxrzError("delete dirs {0} exception:{1}".format(fname,traceback.print_exc()))

def rmfile(fname):
    try:
        if not exists(fname):
            return 0
        else:
            os.remove(fname)
            return 1
    except OSError, oe:
        raise RxrzError("delete file {0} exception:{1}".format(fname,traceback.print_exc()))


def expire(fname, hours=6):
    ntime,ftime = (long(round(time.time()*1000)),sys.float_info.max)
    if hours < 6:
        raise RxrzError('file expire time less 6 hour')
    elif os.path.isfile(fname):
        ftime = long(round(os.path.getmtime(fname)*1000))
        pass
    elif os.path.isdir(fname):
        ftime = long(round(os.path.getctime(fname)*1000))
        pass
    return True if (ntime-ftime)/(1000*60*60) > hours else False

3，任务线程task.py

#!/usr/bin/env python
# encoding: utf-8

##############################################################################
#   Function: 任务线程                            #
#   Author:   why000007@163.com                                             #
#   Date:     09/13/2017                                                     #
##############################################################################

import threading
import time
import os
import logging

from file import get_conf
from fileutil import *

fctime = lambda : time.strftime("%m/%d/%Y %H:%M:%S", time.localtime())
fstime = lambda st_time: time.strftime("%Y%m%d%H%M%S", time.localtime(st_time))
curfday  = lambda : time.strftime("%m/%d/%Y", time.localtime())

class ScanTask(threading.Thread):

    def __init__(self, name, cpath="", exit=False):
        threading.Thread.__init__(self)
        threading.Thread.setName(self,name)
        self.__path = cpath if cpath !="" else get_conf('clean_path',None)
        self.__filetype = get_conf('clean_type',None)
        self.__expire = int(get_conf('expire_day',7))
        self.__interval = int(get_conf('sleep_interval',60))
        self.__exit = exit
        self.reset()

    def reset(self):
        self.day_flag = curfday()
        self.day_files = 0
        self.day_folders = 0
        self.day_fails = 0

    def exit(self): self.__exit = True

    def run(self):
        logging.info("starting %s with interval=%s, expire=%s, path=%s" % (self.getName(), self.__interval, self.__expire,self.__path))
        type_set = eval(self.__filetype)
        loop_flg = False
        while not loop_flg:
            cur_files, cur_folders, cur_fails = (0,0,0)
            if self.day_flag <> curfday():
                self.reset()
            if not exists(self.__path):
                logging.info("cur loop scan not exists path {0}".format(self.__path))
                time.sleep(self.__interval)
                break
            try:
                dir_level1 = 1
                for root, dirs, files in os.walk(self.__path,False):
                    #print "current loop find expire file " + str(len(files))
                    ##############################################################################
                    for file in files:
                        fname = os.path.join(root,file)
                        if istype(fname, type_set) and expire(fname, 24*self.__expire):
                            try:
                                mtime = smtime(fname)
                                dflag = rmfile(fname)
                                logging.info('''delete file[{0}][{1}] {2}'''.format(dflag, fstime(mtime), fname))
                                cur_files += 1
                            except CleanError,re:
                                logging.warn(re.message)
                                cur_fails += 1
                                continue
                    ##############################################################################
                    if str(root) == self.__path:
                        #skip root folder
                        pass
                    elif isempty(root) or expire(root, 24*self.__expire):
                        try:
                            mcime = sctime(root)
                            dflag = rmdirs(root)
                            logging.info('''delete folder[{0}][{1}] {2}'''.format(dflag, fstime(mcime), str(root)))
                            cur_folders += 1
                        except CleanError,re:
                            logging.warn(re.message)
                            cur_fails += 1
                            continue
                    else:
                        #print "current loop find expire folder 0"
                        pass
                    ##############################################################################
            except:
                logging.error(traceback.print_exc())
                continue
            finally:
                self.day_files += cur_files
                self.day_folders += cur_folders
                self.day_fails += cur_fails
                logging.info("cur loop delete expire file {0} ,folder {1} ,fail {2}".format(cur_files,cur_folders,cur_fails))
                logging.info("day loop delete expire file {0} ,folder {1} ,fail {2}".format(self.day_files,self.day_folders,self.day_fails))
                if self.__exit:
                    loop_flg = True
                else:
                    logging.info("{0} sleep {1}s".format(self.getName(),self.__interval))
                    time.sleep(self.__interval)
        #Exiting
        logging.info("exiting thread[%s] at %s" % (self.getName(), fctime()))

4, 日志与配置__init__.py

#!/usr/bin/env python
# encoding: utf-8

##############################################################################
#   Function: 配置与日志                            #
#   Author:   why000007@163.com                                             #
#   Date:     09/11/2017                                                     #
##############################################################################


__all__ = ['file','task']

import os
from ConfigParser import ConfigParser
from logging.handlers import TimedRotatingFileHandler
import logging

###########################################################################################################################
conf_path = os.path.abspath(os.path.join(os.path.dirname(__file__),"..","conf.ini"))
sys_conf = ConfigParser()
sys_conf.read(conf_path)

###########################################################################################################################
handler = TimedRotatingFileHandler(sys_conf.get('sys', 'log_file'), backupCount=7, interval=1, when="D")
handler.setFormatter(logging.Formatter('[%(process)d][%(asctime)s][%(threadName)s][%(levelname)s]-| %(message)s'))
logger = logging.getLogger('')
logger.setLevel(logging.INFO)
logging.getLogger('').addHandler(handler)
console = logging.StreamHandler()
console.setFormatter(logging.Formatter('[%(process)d][%(asctime)s][%(threadName)s][%(filename)s][line:%(lineno)d] %(levelname)s-| %(message)s'))
console.setLevel(logging.DEBUG)
logging.getLogger('').addHandler(console)

###########################################################################################################################
def get_conf(key, default):
    if sys_conf.has_option("sys", key):
        return sys_conf.get("sys", key)
    else:
        return default

class CleanError(Exception):
    def __init__(self, message):
        self.__message = message
    def __str__(self):
        return repr(self.__message)

5, 主函数(入口)

#!/usr/bin/env python
# encoding: utf-8

##############################################################################
#   Function: Schedule Clean Expire File                                     #
#   Author:   why000007@163.com                                              #
#   Date:     09/11/2017                                                     #
##############################################################################

import sys, os
import argparse

parser = argparse.ArgumentParser(description='clean expire file script')
parser.add_argument('--stdin',  type=str, default = '/dev/null')
parser.add_argument('--stdout', type=str, default = '/dev/null')
parser.add_argument('--stderr', type=str, default = '/dev/null')
parser.add_argument('--cpath',  type=str, default = '')
args = parser.parse_args()

from file.task import ScanTask

if __name__ == '__main__':
    cpath = args.cpath
    scanTask = ScanTask("clean-thread", cpath)
    scanTask.start()

6，可通过修改main.py ， 通过修改ScanTask构造参数exit,    然后通过crontab 方式调度执行

 
  
 #自循环Sleep方式
  scanTask = ScanTask("clean-thread", cpath)

#一次任务执行方式
scanTask = ScanTask("clean-thread", cpath， True)