java实现多线程下载文件

2019独角兽企业重金招聘Python工程师标准>>>

//1.1文件下载类：
import java.io.*;
import java.net.*;
import java.util.List;
import fatowen.stocksystem.sysconfig.data.DownLoadVO;
public class HttpDownFile {
 private static int BUFFER_SIZE = 8096;
 /**根据URL下载文件并保存
 * @param destUrl String
 * @param fileName String
 * @throws Exception
 */
 public void saveToFile(String destUrl, String fileName) throws IOException {
 
  FileOutputStream fos = null;
  BufferedInputStream bis = null;
  HttpURLConnection httpUrl = null;
  URL url = null;
  byte[] buf = new byte[BUFFER_SIZE];
  int size = 0;
 
  url = new URL(destUrl);
  httpUrl = (HttpURLConnection) url.openConnection();
  httpUrl.connect();
  bis = new BufferedInputStream(httpUrl.getInputStream());
  fos = new FileOutputStream(fileName);
  while ((size = bis.read(buf)) != -1)
   fos.write(buf, 0, size);
  fos.close();
  bis.close();
  httpUrl.disconnect();
 }
}


//1.2多线程实现下载类：

import java.util.ArrayList;
import java.util.List;
public class HisDataAddThread extends Thread {
 boolean runFlag = true;
 List myParamList = null;
 String downLoadData ="";
 String baseUrl = "http://table.finance.yahoo.com/table.csv?s=";
 String result = "";
 String savePath = "";
 
 public HisDataAddThread(List paramList,String savePath){
  this.myParamList = paramList;
  this.savePath = savePath;
 }
 
 public void run() {
 
  while(runFlag){
   downLoadData = PublicDataUtil.getDownLoadData(myParamList);
   if(!Lib.isEmpty(downLoadData)){
    HttpDownFile oInstance = new HttpDownFile();
    try {
     oInstance.saveToFile(baseUrl + downLoadData, savePath + downLoadData + ".csv");
    }catch (Exception err) {
     System.out.println(err.toString());
    }
   }else{
    runFlag = false;
   }
   try {
    Thread.sleep(1000);
   } catch (InterruptedException e) {
    e.printStackTrace();
   }
  }
 }
 public List getFailureList() {
  return failureList;
 }
 public void setFailureList(List failureList) {
  this.failureList = failureList;
 }
 public List getSuccessList() {
  return successList;
 }
 public void setSuccessList(List successList) {
  this.successList = successList;
 }
}

//2.将下载完的文件统一保存到数据库工具类
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
public class CSVUtitl {
 private BufferedReader bufferedreader = null;
 private List list = new ArrayList();
 
 public CSVUtitl(){
 }
 public CSVUtitl(String filename) throws IOException{
       bufferedreader = new BufferedReader(new FileReader(filename));
       String stemp;
       while((stemp = bufferedreader.readLine()) != null){
           list.add(stemp);
       }
 }
 public List getList() throws IOException {
        return list;
 }
 // 得到csv文件的行数
 public int getRowNum(){
     return list.size();
 }
    //得到csv文件的列数
 public int getColNum(){
       if(!list.toString().equals("[]")) {
      
         //csv文件中，每列之间的是用','来分隔的
            if(list.get(0).toString().contains(",")) { 
                return list.get(0).toString().split(",").length;
            }else if(list.get(0).toString().trim().length() != 0) {
                return 1;
            }else{
                return 0;
            }
       }else{
            return 0;
        }
 }
 
    //取得指定行的值
 public String getRow(int index) {
     if (this.list.size() != 0)
      return (String) list.get(index);
     else                      
      return null;
 }
 //取得指定列的值
 public String getCol(int index){
       if (this.getColNum() == 0){
                return null;
       }
      
       StringBuffer scol = new StringBuffer();
       String temp = null;
       int colnum = this.getColNum();
     
       if (colnum > 1){
          for (Iterator it = list.iterator(); it.hasNext();) {
             temp = it.next().toString();
             scol = scol.append(temp.split(",")[index] + ",");
          }
       }else{
          for (Iterator it = list.iterator(); it.hasNext();) {
            temp = it.next().toString();
            scol = scol.append(temp + ",");
          }
       }
       String str=new String(scol.toString());
       str = str.substring(0, str.length() - 1);
       return str;
 }
 //取得指定行，指定列的值
 public String getString(int row, int col) {
        String temp = null;
        int colnum = this.getColNum();
        if(colnum > 1){
            temp = list.get(row).toString().split(",")[col];
        }else if(colnum == 1) {
            temp = list.get(row).toString();
        }else{
            temp = null;
        }
            return temp;
 }

 public void CsvClose() throws IOException {
     this.bufferedreader.close();
 }
 public void run(String filename) throws IOException {
        
  CSVUtitl cu = new CSVUtitl(filename);
     for(int i=0;i<cu.getRowNum();i++){
           String SSCCTag = formatData(cu.getString(i,1));//得到第i行.第一列的数据.
           String SiteName = formatData(cu.getString(i,2));//得到第i行.第二列的数据.
           String StationId= formatData(cu.getString(i,3));
          
           //将数据保存到数据库中
           ... ...
           ... ...
           ... ...
     }
     cu.CsvClose();
 }
 public String formatData(String baseData){
 
  String result = null;
  if(!"".equals(baseData) && baseData != null){
   if(baseData.length() > 1){
    result = baseData.substring(1,baseData.length());
    result = result.substring(0, result.length()-1);
   }else{
    result = baseData;
   }
  }else{
   result = "";
  }
  return result.trim();
 }
 
 public static void main(String[] args) throws IOException {
  CSVUtitl test = new CSVUtitl();
  try{
   File path = new File("e:\\data");
   File[] f = path.listFiles();
   List l = new ArrayList();
   for(int i=0;i<f.length;i++){
    if(f[i].getName().endsWith(".csv"))
    l.add(f[i]); www.2cto.com
   }
   Iterator it = l.iterator();
   while(it.hasNext()){
    File ff = (File)it.next();
    test.run(path.toString()+File.separator+ff.getName());
   }
         }catch (Exception e){
     
   }
  }
}

转载于:https://my.oschina.net/icelily/blog/78447