多线成下载数据

package com.cjnetwork.tiles.aatest;

import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.jdom.Document;
import org.jdom.Element;
import org.jdom.input.SAXBuilder;
import org.xml.sax.InputSource;


import com.cjnetwork.tiles.model.Lnglat;
import com.cjnetwork.tiles.model.MapParameter;
import com.cjnetwork.tiles.model.Tile;
import com.cjnetwork.tiles.model.TilesRange;
import com.cjnetwork.tiles.mongo.MongoService;
import com.cjnetwork.tiles.mongo.MongodServiceImpl;
import com.cjnetwork.tiles.util.ConfigUtil;
import com.cjnetwork.tiles.util.CoordinateUtil;
import com.cjnetwork.tiles.util.DownloadUtil;

/**
 * <p>file name: Test.java</p>
 * <p>despription: </p>
 */
public class Test {

	static ExecutorService pool;
	static String downloadDir;
	static int[] zoom;
	static Lnglat leftTopLnglat;
	static Lnglat rightBottomLnglat;
	static int roundCount;
	static int totalSize;
	static int currentIndex = 0;
	static int failedCount = 0;
	static ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
	static ReentrantReadWriteLock lockFailCount = new ReentrantReadWriteLock();
	//static MongodServiceImpl mongoDAO = new MongoService("test", "fs");
	static MapParameter mapParameter=new MapParameter();
	/**
	 * 
	 * 功能描述：<br>
	 * 当前处理索引加1
	 * 
	 * @return void
	 * 
	 * 修改记录:
	 */
	public static int addCurrentIndex(){
		synchronized (lock.writeLock()) {
			try{
				lock.writeLock().lock();
				currentIndex++;
			}finally{
				lock.writeLock().unlock();
			}
		}
		return currentIndex;
	}
	
	public static void addFailedCount(){
		synchronized (lockFailCount.writeLock()) {
			try{
				lockFailCount.writeLock().lock();
				failedCount++;
			}finally{
				lockFailCount.writeLock().unlock();
			}
		}
	}
	public static int getFailedCount(){
		int count = 0;
		synchronized (lockFailCount.readLock()) {
			try{
				lockFailCount.readLock().lock();
				count = failedCount;
			}finally{
				lockFailCount.readLock().unlock();
			}
		}
		return count;
	}
	
	
	public static void main(String[] args) {
		init();
		Test test=new Test();
		test.getMapServerUrl();
		boolean isCustomExtent=false;
		isCustomExtent= Boolean.valueOf(ConfigUtil.get("isCustomExtent"));
		if(!isCustomExtent){
			String[] dataBounds=mapParameter.getDataBounds().trim().split(",");
			leftTopLnglat= new Lnglat(Double.valueOf(dataBounds[0].trim()), Double.valueOf(dataBounds[3].trim()));
			rightBottomLnglat = new Lnglat(Double.valueOf(dataBounds[2].trim()), Double.valueOf(dataBounds[1].trim()));
			System.out.println("左上角:"+leftTopLnglat);
			System.out.println("右下角:"+rightBottomLnglat);
		}else{
			String[] tmpLngLat = ConfigUtil.get("leftTopLnglat").split(",");
			leftTopLnglat = new Lnglat(Double.valueOf(tmpLngLat[0].trim()), Double.valueOf(tmpLngLat[1].trim()));
			tmpLngLat = ConfigUtil.get("rightBottomLnglat").split(",");
			rightBottomLnglat = new Lnglat(Double.valueOf(tmpLngLat[0].trim()), Double.valueOf(tmpLngLat[1].trim()));
		}
		printCalculatSize(zoom, leftTopLnglat, rightBottomLnglat);
		//lonLatsStartDownload(zoom, leftTopLnglat, rightBottomLnglat);
		List<Tile> tmpTileList = new ArrayList<Tile>();
		int tmpTileListSize = 0;
		for (int i = 0; i < zoom.length; i++) {
			TilesRange sss=test.GetRowColValue(leftTopLnglat.getLng(),rightBottomLnglat.getLat(),rightBottomLnglat.getLng(),leftTopLnglat.getLat(),zoom[i]);
			 for(int j=sss.getMinCol();j<=sss.getMaxCol();j++){
				 for(int k=sss.getMinRow();k<=sss.getMaxRow();k++){
	
					 //System.out.println("j:="+j+",k:="+k+",zoom[i]:="+zoom[i]);
					 startDownloadThreadTile(new Tile(j, k, zoom[i]));
//					    tmpTileList.add(new Tile(j, k, zoom[i]));
//					    tmpTileListSize++;
//						if(tmpTileListSize >= roundCount){
//							tmpTileListSize = 0;
						   //startDownloadThread(tmpTileList);
						//}
				 }
			 }
		}
		//System.out.println("tmpTileList:"+tmpTileList.size());
		if(tmpTileListSize != 0){
			tmpTileListSize = 0;
		//	startDownloadThread(tmpTileList);
		}
	}

	/**
	 * 
	 * 功能描述：<br>
	 * 初始化
	 * 
	 * @return void
	 * 
	 * 修改记录:
	 */
	private static void init() {
		try{
			boolean isProxy = false;
			isProxy = Boolean.valueOf(ConfigUtil.get("isProxy"));
			if(isProxy){
				DownloadUtil.setProxy(isProxy);
				DownloadUtil.setProxyAddress(ConfigUtil.get("proxyAddress"));
				try{DownloadUtil.setProxyPort(Integer.parseInt(ConfigUtil.get("proxyPort")));}catch (Exception e) {}
			}
		}catch (Exception e) {}
		downloadDir = ConfigUtil.get("downloadDir");
		
		pool = Executors.newFixedThreadPool(Integer.parseInt(ConfigUtil.get("threadCount")));
		roundCount = Integer.parseInt(ConfigUtil.get("roundCount").trim());
		String[] tmpZoom = ConfigUtil.get("zoom").split(",");
		System.out.println(tmpZoom.toString());
		zoom = new int[tmpZoom.length];
		for(int i = 0, len = zoom.length; i < len; i++){
			zoom[i] = Integer.parseInt(tmpZoom[i].trim());
		}
		
		
	}
	
	/*
	 * 解析栅格地图服务端的参数
	 */
	public  void getMapServerUrl(){
		//System.out.println(ConfigUtil.get("mapServerUrl"));
		String tempUrl=ConfigUtil.get("mapServerUrl");
		String mapServer=tempUrl.substring(0,tempUrl.indexOf("EHL_MapServer"))+"EHL_MapServer/servlet/MapServlet?type=getMapInfo";
        //下载mapParameter.xml参数文件
		String savePath=downloadDir+File.separatorChar+"temp"+File.separatorChar+"mapParameter.xml";
		if(mapServer!=null){
			try {
				DownloadUtil.download(mapServer, savePath, false);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		URL   url   =   null;
		 HttpURLConnection   httpurlconnection   =   null; 
		 try {
			url=new URL(mapServer);
		     // 以post方式请求
	         httpurlconnection =(HttpURLConnection)url.openConnection();
	         httpurlconnection.setRequestProperty("contentType", "GBK"); 
	         httpurlconnection.setDoOutput(true);
	         httpurlconnection.setRequestMethod("POST");
	         // 获取响应代码
	         int code   = httpurlconnection.getResponseCode();   
	         // 获取页面内容
	         BufferedReader in = null;
	         StringBuffer sb = new StringBuffer();
	         in = new BufferedReader(new InputStreamReader(httpurlconnection
	                 .getInputStream(),"UTF-8"));
	         String inputLine;
	         while ((inputLine = in.readLine()) != null) {
	             sb.append(inputLine);
	         }
	         System.out.println(sb.toString());
	         StringReader read = new StringReader(sb.toString());
	         InputSource source = new InputSource(read);
	         SAXBuilder sax = new SAXBuilder();
	            Document doc = sax.build(source);
	            Element root = doc.getRootElement();
	            mapParameter.setMapspanscale(root.getChildText("mapspanscale"));
	            mapParameter.setNCoord( root.getChildText("nCoord"));
	            mapParameter.setNProjection(root.getChildText("nProjection"));
	            mapParameter.setBCustomResolution( root.getChildText("bCustomResolution"));
	            mapParameter.setBaseResolution( root.getChildText("baseResolution"));
	            mapParameter.setXOrigin( root.getChildText("xOrigin"));
	            mapParameter.setYOrigin( root.getChildText("yOrigin"));
	            mapParameter.setDFalseEasting(root.getChildText("dFalseEasting"));
	            mapParameter.setDFalseNorthing(root.getChildText("dFalseNorthing") );
	            mapParameter.setDCentralMeridian( root.getChildText("dCentralMeridian"));
	            mapParameter.setDLatitudeOfOrigin(root.getChildText("dLatitudeOfOrigin"));
	            mapParameter.setStandardParallel1( root.getChildText("standardParallel1"));
	            mapParameter.setStandardParallel2( root.getChildText("standardParallel2"));
	            mapParameter.setMapUnits( root.getChildText("mapUnits"));
	            mapParameter.setMapSize( root.getChildText("MapSize"));
	            mapParameter.setImageFormat( root.getChildText("imageFormat"));
	            mapParameter.setBound(root.getChildText("bounds"));
	            mapParameter.setDataBounds(root.getChildText("dataBounds"));
		} catch (Exception e) {
			e.printStackTrace();
		}   finally{
	         if(httpurlconnection!=null)  
	             httpurlconnection.disconnect();   
	     }
		
	}

	private static void printCalculatSize(int[] zoom, Lnglat leftTopLnglat, Lnglat rightBottomLnglat) {
		//int size = calculateDownloadSize(zoom, leftTopLnglat, rightBottomLnglat);
		//System.out.println("下载tile数量:" + size);
		//System.out.println("下载大小:" + size * 22 + "k, " + size * 22 / 1024 + "M, " + size * 22 / 1024 / 1024 + "G");
	}

	private static int calculateDownloadSize(int[] zoom, Lnglat leftTopLnglat, Lnglat rightBottomLnglat) {
		int size = 0;
		for(int i = 0, len = zoom.length; i < len; i++){
			Tile leftTopTile = CoordinateUtil.lnglatToTile(zoom[i], leftTopLnglat);
			Tile rightBottomTile = CoordinateUtil.lnglatToTile(zoom[i], rightBottomLnglat);
			size += (rightBottomTile.getX() - leftTopTile.getX() + 1) * (rightBottomTile.getY() - leftTopTile.getY() + 1);
		}
		return size;
	}
	/*
	 * 补充在芒果数据库中没有下载完的图片通过下载工具重新增加补充到芒果库中，主要是针对魔卡托投影
	 */
	/*
	private static void startDownload(int[] zoom, Lnglat leftTopLnglat, Lnglat rightBottomLnglat) {
		totalSize = calculateDownloadSize(zoom, leftTopLnglat, rightBottomLnglat);
		List<Tile> tmpTileList = new ArrayList<Tile>();
		int tmpTileListSize = 0;
		for(int i = 0, len = zoom.length; i < len; i++){
			Tile leftTopTile = CoordinateUtil.lnglatToTile(zoom[i], leftTopLnglat);
			Tile rightBottomTile = CoordinateUtil.lnglatToTile(zoom[i], rightBottomLnglat);
			for(int x = leftTopTile.getX();x <= rightBottomTile.getX(); x++){
				for(int y = leftTopTile.getY();y <= rightBottomTile.getY(); y++){
					tmpTileList.add(new Tile(x, y, zoom[i]));
					//System.out.println("row:"+x+","+"col:"+y+",zoom:"+zoom[i]);//获取行列号，以及级别
					String fileName=zoom[i]+"-"+y+"-"+x+".png";
					if(mongoDAO.findFileByName(fileName)==null){
					
					    try {
					    	//URL url = new URL(tempUrl);
					    	mongoDAO.saveFile(new URL(tempUrl).openStream(), fileName);
					    	//System.out.println(ImageIO.read(new URL(tempUrl)).getHeight());
						} catch (Exception e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						} 
						System.out.println(tempUrl);
						System.out.println("文件名："+fileName);
					}
					tmpTileListSize++;
					if(tmpTileListSize >= roundCount){
						tmpTileListSize = 0;
					//startDownloadThread(tmpTileList);
					}
				}
			}
		}
		if(tmpTileListSize != 0){
			tmpTileListSize = 0;
			//startDownloadThread(tmpTileList);
		}
	}*/
	
	/**
	 * 
	 * 功能描述：<br>
	 * 为了解决arraylist 超过最大长度报错的问题
	 * 循环进行单进程下载
	 * @param Tile
	 * @return void
	 * 修改记录:
	 */
	private static void startDownloadThreadTile(Tile tile){
		try {
			downloadXYZ(tile);
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	
	
	
	/**
	 * 
	 * 功能描述：<br>
	 * 开始下载线程
	 * 
	 * @param tmpTileList
	 * @return void
	 * 
	 * 修改记录:
	 */
	private static void startDownloadThread(List<Tile> tmpTileList) {
		final Tile[] threadTaskTiles = tmpTileList.toArray(new Tile[0]);
		tmpTileList.clear();
		pool.execute(new Runnable() {
			@Override
			public void run() {
				for(int i = 0, len = threadTaskTiles.length; i < len; i++){
					try{
						System.out.println("进度：第 " + addCurrentIndex() + " 个，总 " + totalSize + " 个，失败 " + Test.getFailedCount() + " 个");
						downloadXYZ(threadTaskTiles[i]);
					}catch (Exception e) {
						Test.addFailedCount();
						e.printStackTrace();
						
						System.out.println("下载失败:x(" + threadTaskTiles[i].getX() + "),y(" + threadTaskTiles[i].getY() + "),z(" + threadTaskTiles[i].getZoom() + ")");
					}
				}
			}
		});
	}
	
	/**
	 * 
	 * 功能描述：<br>
	 * 经纬度直投
	 * 
	 * @param tmpTileList
	 * @return void
	 * 
	 * 修改记录:
	 */
	private static void lonLatsStartDownload(int[] zoom, Lnglat leftTopLnglat, Lnglat rightBottomLnglat) {
		totalSize = calculateDownloadSize(zoom, leftTopLnglat, rightBottomLnglat);
		List<Tile> tmpTileList = new ArrayList<Tile>();
		int tmpTileListSize = 0;
		for(int i = 0, len = zoom.length; i < len; i++){
			Tile leftTopTile = CoordinateUtil.lnglatToTile(zoom[i], leftTopLnglat);
			Tile rightBottomTile = CoordinateUtil.lnglatToTile(zoom[i], rightBottomLnglat);
			
			for(int x = leftTopTile.getX();x <= rightBottomTile.getX(); x++){
				for(int y = leftTopTile.getY();y <= rightBottomTile.getY(); y++){
					tmpTileList.add(new Tile(x, y, zoom[i]));
					//System.out.println("row:"+x+","+"col:"+y+",zoom:"+zoom[i]);//获取行列号，以及级别
					String tempUrl="http://localhost:8080/EHL_MapServer/DCMap?mapname=Sate"+"&Service=getImage&Zoom="+zoom[i]+"&x="+y+"&y="+x;
					System.out.println(tempUrl);
					/*
					if(mongoDAO.findFileByName(fileName)==null){
					 //根据ehl——mapclient的规则进行读取图片url路径：
						String tempUrl="http://10.2.25.12:8080/EHL_MapServer/DCMap?mapname=Sate"+"&Service=getImage&x="+x+"&y="+y+"&Zoom="+zoom[i];
					    try {
					    	//URL url = new URL(tempUrl);
					    	mongoDAO.saveFile(new URL(tempUrl).openStream(), fileName);
					    	//System.out.println(ImageIO.read(new URL(tempUrl)).getHeight());
						} catch (MalformedURLException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						} catch (IOException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						}
						System.out.println(tempUrl);
						//http://localhost:8080/EHL_MapServer/DCMap?mapname=Sate+"&Service=getImage&x="+tile[0]+"&y="+tile[1]+"&Zoom="+zoom;
						System.out.println("文件名："+fileName);
					}*/
				
					
					
					tmpTileListSize++;
					if(tmpTileListSize >= roundCount){
						tmpTileListSize = 0;
						
					startDownloadThread(tmpTileList);
					}
				}
			}
		}
		if(tmpTileListSize != 0){
			tmpTileListSize = 0;
			startDownloadThread(tmpTileList);
		}
	}
	
	
	private static void downloadXYZ(Tile tile) throws Exception{
		
		//String url = "http://mt0.googleapis.com/vt?src=apiv3&x=" + tile.getX() + "&y=" + tile.getY() + "&z=" + tile.getZoom();//矢量
		//String url = "http://mt2.google.cn/vt/lyrs=s@142&hl=zh-CN&gl=CN&src=app&s=G&x=" + tile.getX() + "&y=" + tile.getY() + "&z=" + tile.getZoom();
		String url=ConfigUtil.get("mapServerUrl").trim()+"&Service=getImage&Zoom="+ tile.getZoom()+"&x="+tile.getX()+"&y="+tile.getY() ;
		String storePath = downloadDir + "/tiles/" + tile.getZoom() + "/" + tile.getY()+ "/" +tile.getX()  + ".png";
		storePath = storePath.replace("//", "/");
	    DownloadUtil.download(url, storePath, false);
	}
}