数据库大数据量导出多线程版本源码部分

package com.alibaba.crm.finance.bo.export;

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.SQLException;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.log4j.Logger;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.BeanFactoryAware;

import com.ali.shy.dao.Queryable;
import com.ali.shy.util.JSON;
import com.ali.shy.util.json.ParseException;
import com.alibaba.crm.finance.biz.common.BizConstant;
import com.alibaba.crm.finance.biz.export.CompressedFilesUtil;
import com.alibaba.crm.finance.biz.export.NotepadRender;
import com.alibaba.crm.finance.common.log.LogModel;
import com.alibaba.crm.finance.common.log.LogUtil;
import com.alibaba.crm.finance.dataobject.base.sys.FinExportTask;

/**
 * @author longer 2012-4-16 \u4e0b\u53485:04:55
 */
@SuppressWarnings({ "unchecked" })
public class QueueGenerateExcel implements BeanFactoryAware, ExportStartNow {

    private FinExportTaskBo        finExportTaskBo;

    private NotepadRender          notepadRender;

    private static final Logger    LOG              = Logger.getLogger(QueueGenerateExcel.class);

    private static final Object    LOCK             = new Object();
    private static final int       THREAD_POOL_SIZE = 3;

    private static ExecutorService EXECUTOR_SERVICE = null;
    static {
        EXECUTOR_SERVICE = Executors.newFixedThreadPool(THREAD_POOL_SIZE, new ThreadFactory() {

            final AtomicInteger threadNumber = new AtomicInteger(1);

            public Thread newThread(Runnable r) {
                ThreadGroup group = Thread.currentThread().getThreadGroup();
                Thread t = new Thread(group, r, "export-pool-" + threadNumber.getAndIncrement(), 0);
                return t;
            }
        });
    }

    public void setNotepadRender(NotepadRender notepadRender) {
        this.notepadRender = notepadRender;
    }

    private static BeanFactory mFact;

    public void setBeanFactory(BeanFactory beanFactory) {
        mFact = beanFactory;
    }

    public void setFinExportTaskBo(FinExportTaskBo finExportTaskBo) {
        this.finExportTaskBo = finExportTaskBo;
    }

    /*
     * (non-Javadoc)
     * @see com.alibaba.crm.finance.bo.export.ExportStartNow#startExportNow()
     */
    public void startExportNow() {
        synchronized (LOCK) {
            LOCK.notifyAll();
        }
    }

    public void init() {
        try {
            for (int i = 0; i < THREAD_POOL_SIZE; i++) {
                EXECUTOR_SERVICE.execute(new Generate(i));
            }
            EXECUTOR_SERVICE.shutdown();
        } catch (Exception e) {
            LogModel model = new LogModel("QueueGenerateExcel", "\u5bfc\u51fa\u5b9a\u65f6\u949f", e);
            LogUtil.error(LOG, model);
        }
    }

    public void close() {
        EXECUTOR_SERVICE.shutdownNow();
    }

    private class Generate extends Thread {

        private static final int  COMPUTE_TIME_TIMES        = 10;
        private static final int  DEFFER_MIN_PAGE_SIZE      = 1000;
        private static final int  ONE_SECOND                = 1 * 1000;
        private static final int  PAGE_PRE_SIZE             = 4000;

        private static final int  WAIT_TIMEOUT_FOR_OUT_NULL = 1000;
        private static final long WAIT_TIMEOUT_FOR_NULL     = 2 * 60 * 1000L;

        public Generate(int i){
            super("Generate_export_" + i);
        }

        public void run() {
            LogModel model = new LogModel("QueueGenerateExcel Generate run");
            while (Boolean.TRUE) {
                FinExportTask finExportTask = null;
                try {
                    finExportTask = loadNextTask();
                    if (finExportTask != null) {
                        deleteBeforeFile(finExportTask.getFileName());
                        if (executeOneTask(finExportTask)) {
                            finExportTaskBo.finishTheTask(finExportTask.getId(), finExportTask.getFileName(),
                                                          finExportTask.getFileSize());
                            LogUtil.logInfo(model, LOG, finExportTask + " finish!!");
                        } else {
                            LogUtil.logInfo(model, LOG, finExportTask + " break!!");
                        }

                    }
                    sleep(finExportTask);
                } catch (Exception e) {
                    if (finExportTask != null) {
                        finExportTask.setServiceEndDate(new Date());
                        finExportTaskBo.failureTheTsak(finExportTask.getId());
                    }
                    LogUtil.logError(model, LOG, e.getMessage(), e);
                }
            }
        }

        private void deleteBeforeFile(String fileName) {
            LogModel model = new LogModel("QueueGenerateExcel Generate deleteBeforeFile");
            try {
                LogUtil.logInfo(model, LOG, "deleteBeforeFile START");
                File f = new File(fileName);
                if (f.exists()) {
                    f.renameTo(new File(fileName + "." + new Date().getTime() + ".back"));
                }
                File frar = new File(fileName + CompressedFilesUtil.FILE_STYLE);
                if (frar.exists()) {
                    frar.renameTo(new File(fileName + CompressedFilesUtil.FILE_STYLE + "." + new Date().getTime()
                                           + ".back"));
                }
                LogUtil.logInfo(model, LOG, "deleteBeforeFile END");
            } catch (Exception e) {
                LogUtil.logError(model, LOG, e.getMessage(), e);
            }
        }

        private void sleep(FinExportTask finExportTask) throws InterruptedException {
            if (finExportTask == null) {
                synchronized (LOCK) {
                    LOCK.wait(WAIT_TIMEOUT_FOR_NULL);// wait 2min
                }
            } else {
                Thread.sleep(WAIT_TIMEOUT_FOR_OUT_NULL);// sleep 1s
            }
        }

        private FinExportTask loadNextTask() throws InterruptedException {
            LogModel model = new LogModel("loadNextTask");
            while (Boolean.TRUE) {
                FinExportTask finExportTask = finExportTaskBo.loadNextTask();
                if (finExportTask == null) {
                    LogUtil.logInfo(model, LOG, "all finExportTask has  finish !");
                    return finExportTask;
                }
                if (finExportTaskBo.startTheTask(finExportTask)) {
                    finExportTask.setStatus(BizConstant.EXPORT_TASK_STATUS_SERVICE);
                    finExportTask.setServiceBeginDate(new Date());
                    LogUtil.logInfo(model, LOG, finExportTask + " start!");
                    return finExportTask;
                } else {
                    LogUtil.logInfo(model, LOG, finExportTask + " is in service!");
                }
                Thread.sleep(ONE_SECOND);
            }
            return null;
        }

        private Export genExport(FinExportTask finExportTask) throws ParseException {
            Export e = new Export();
            String json = finExportTask.getSearchCondition();
            Map<String, Object> data = (Map<String, Object>) JSON.parse(json);

            String fileName = finExportTask.getFileName();
            String action = String.valueOf(data.get("dao"));

            String query = String.valueOf(data.get("query"));
            if ("null".equals(query)) {
                query = null;
            }
            int max = Integer.valueOf(String.valueOf(data.get("max")));
            int skip = Integer.valueOf(String.valueOf(data.get("skip")));
            e.setSkip(skip);
            e.setAction(action);
            e.setFileName(fileName);
            e.setMax(max);
            e.setQuery(query);
            e.setData(data);
            return e;
        }

        private Boolean executeOneTask(FinExportTask finExportTask) throws ParseException, SQLException, IOException,
                                                                   InterruptedException {
            LogModel model = new LogModel("executeOneTask");
            Export e = genExport(finExportTask);
            reviseMax(e);
            int pageSize = e.getMax() / PAGE_PRE_SIZE + 1;// \u9875\u6570
            if ((e.getMax() % PAGE_PRE_SIZE) == 0) {
                pageSize = pageSize - 1;
            }

            Queue<Long> times = new LinkedList<Long>();
            for (int i = 0; i < pageSize; i++) {
                if (breakTask(finExportTask)) {
                    LogUtil.logInfo(model, LOG, finExportTask + " break!!");
                    return Boolean.FALSE;
                }
                int skipp = e.getSkip() + i * PAGE_PRE_SIZE;
                int pagePreSize = PAGE_PRE_SIZE;
                if (i == pageSize - 1) {
                    pagePreSize = e.getMax() - (pageSize - 1) * PAGE_PRE_SIZE;
                    // FOR DEEFER
                    if (pagePreSize <= DEFFER_MIN_PAGE_SIZE && e.getAction().equals("defferDaoModelAction")) {
                        pagePreSize = DEFFER_MIN_PAGE_SIZE + 1;
                    }
                }
                long startTime = new Date().getTime();
                List<Object> datas = getDao(e.getAction()).queryForList(e.getQuery(), e.getData(), skipp, pagePreSize);
                notepadRender.trans(datas, new File(e.getFileName()));

                computeRemainingTime(finExportTask, pageSize, times, i, startTime);
                // for compatibility
                if (datas == null || datas.size() == 0) {
                    break;
                }
                datas.clear();
            }
            // for compressed
            BigDecimal size = CompressedFilesUtil.compressedFiles(new File(e.getFileName()));
            finExportTask.setFileSize(size);
            finExportTask.setFileName(finExportTask.getFileName() + CompressedFilesUtil.FILE_STYLE);
            return Boolean.TRUE;
        }

        private boolean breakTask(FinExportTask finExportTask) {
            FinExportTask finExportTasknew = finExportTaskBo.getFinExportTask(finExportTask.getId());
            if (!BizConstant.EXPORT_TASK_STATUS_SERVICE.equals(finExportTasknew.getStatus())) {
                return Boolean.TRUE;
            } else {
                return Boolean.FALSE;
            }
        }

        private void computeRemainingTime(FinExportTask finExportTask, int pageSize, Queue<Long> times, int i,
                                          long start) {
            long end = new Date().getTime();
            long time = (pageSize - i) * (end - start);
            if (times.size() > COMPUTE_TIME_TIMES) {
                times.remove();
            }
            times.add(time);
            long timeall = 0;
            for (Long lo : times) {
                timeall += lo;
            }
            time = timeall / times.size();
            finExportTask.setServiceEndDate(new Date(new Date().getTime() + time));
            finExportTaskBo.computeRemainingTime(finExportTask);
        }

        private void reviseMax(Export e) throws SQLException {
            int skip = e.getSkip();
            String action = e.getAction();
            String query = e.getQuery();
            int max = e.getMax();
            int maxTo = getDao(action).queryCount(query, e.getData());
            if (maxTo < max + skip) {
                max = maxTo - skip + 1;
                if (max < 0) {
                    max = 0;
                }
            }
            e.setMax(max);
        }

        private Queryable getDao(String dao) throws SQLException {

            Object ret = mFact.getBean(dao);
            if (ret == null) {
                throw new SQLException("Dao bean [" + dao + "] not found.");
            }
            if (!(ret instanceof Queryable)) {
                throw new SQLException("Dao bean [" + dao + "] must implements com.ali.shy.dao.Queryable.");
            }
            return (Queryable) ret;
        }

        private final class Export {

            public String getFileName() {
                return fileName;
            }

            public void setFileName(String fileName) {
                this.fileName = fileName;
            }

            public String getAction() {
                return action;
            }

            public void setAction(String action) {
                this.action = action;
            }

            public String getQuery() {
                return query;
            }

            public void setQuery(String query) {
                this.query = query;
            }

            public int getSkip() {
                return skip;
            }

            public void setSkip(int skip) {
                this.skip = skip;
            }

            public int getMax() {
                return max;
            }

            public void setMax(int max) {
                this.max = max;
            }

            private String fileName;
            private String action;
            private String query;
            private int    skip;
            private int    max;

            public Map<String, Object> getData() {
                return data;
            }

            public void setData(Map<String, Object> data) {
                this.data = data;
            }

            private Map<String, Object> data;
        }
    }
}