java版迅雷下载源码分享

最新推荐文章于 2024-07-03 02:53:36 发布

转载最新推荐文章于 2024-07-03 02:53:36 发布 · 6.8k 阅读

本文介绍了一个使用Java实现的支持多任务、多线程下载、断点续传等功能的下载工具。利用HTTP Range请求头技术，实现了文件的分块下载，并通过多线程并发提高下载效率。

这段时间研究java的io与nio框架，一时兴起决定用java实现一个下载工具，主要有下面几个功能

1)支持多任务下载

2）支持多线程下载

3) 支持断点续传

4)错误线程任务的重新调度

用到的技术点

1) http协议的range头技术

2)java的多线程

3)java的网络编程，主要是HttpUrlConnection类

4)java的io文件操作, 如RandomAccessFile类

速度还不错，基本上和浏览器下载差不多，可能比它还要快，我只开了10个线程。

package org.blackfoxer.cat;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLDecoder;
import java.text.DecimalFormat;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Job {
private int fileSize;
private String fileName;
private int connectTimeout = 10000;
private int readTimeout = 20000;
private String url;
private String storeDir;
private int taskNum;
private String jobId;
private int[] startIndexes;
private int[] endIndexes;
private int[] progress;
private Task[] tasks;
private File storeDirFile;
private File dtDirFile;
private File localFile;
private ThreadLocal<RandomAccessFile> rafLocalTl;
private ThreadLocal<RandomAccessFile> rafOffsetTl;
private CountDownLatch latch;
private ProgressThread pt;
public Job(String url, String storeDir, int taskNum) throws IOException {
this.url = url;
this.storeDir = storeDir;
this.taskNum = taskNum;
this.startIndexes = new int[taskNum];
this.endIndexes = new int[taskNum];
this.progress = new int[taskNum];
this.tasks = new Task[taskNum];
this.latch = new CountDownLatch(taskNum);
this.jobId = Math.abs(url.hashCode()) + "_" + taskNum;
this.rafLocalTl = new ThreadLocal<RandomAccessFile>();
this.rafOffsetTl = new ThreadLocal<RandomAccessFile>();
this.pt = new ProgressThread();
}
public void startJob() throws Exception {
long start = System.currentTimeMillis();
System.out.println("开始下载文件...");
boolean j = fetchFileMetaInfo();
if (j) {
assignTasks();
createFiles();
startTasks();
openProgressThread();
waitForCompeletion();
long end = System.currentTimeMillis();
System.out.println("下载完毕,全程耗时" + (end - start) + "ms");
} else {
System.out.println("获取文件长度或文件名失败，请重试");
}
}
private void openProgressThread() {
this.pt.start();
}
private void waitForCompeletion() throws Exception {
latch.await();
deleteFiles();
pt.join();
}
private void deleteFiles() {
if (dtDirFile != null) {
File[] subFiles = dtDirFile.listFiles();
for (File subFile : subFiles) {
subFile.delete();
}
dtDirFile.delete();
}
}
// 1.fetch file size and file name
private boolean fetchFileMetaInfo() throws IOException {
HttpURLConnection connection = createConnection();
connection.setRequestMethod("GET");
if (connection.getResponseCode() == 200) {
this.fileSize = connection.getContentLength();
String disposition = connection.getHeaderField("Content-Disposition");
if (disposition == null) {
parseFileNameFromUrl(url);
} else {
parseFileNameFromDisposition(disposition);
}
if (this.fileName == null || this.fileSize < 0) {
return false;
}
System.out.println("找到文件资源，长度为" + fileSize + ",资源名称为" + fileName);
return true;
}
return false;
}
private void parseFileNameFromUrl(String url) throws UnsupportedEncodingException {
this.fileName = url.substring(url.lastIndexOf("/") + 1, url.length());
if (this.fileName.contains("%")) {
this.fileName = URLDecoder.decode(this.fileName, "UTF-8");
}
}
private void parseFileNameFromDisposition(String disposition) throws UnsupportedEncodingException {
Pattern pattern = Pattern.compile(".+filename=\"(.+?)\".*");
Matcher matcher = pattern.matcher(disposition);
if (matcher.matches()) {
this.fileName = new String(matcher.group(1).getBytes("ISO-8859-1"), "UTF-8");
} else {
parseFileNameFromUrl(url);
}
}
public HttpURLConnection createConnection() throws IOException {
URL urlObj = new URL(url);
HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
connection.setConnectTimeout(connectTimeout);
connection.setReadTimeout(readTimeout);
connection.setRequestProperty("Accept-Charset", "UTF-8");
connection.setRequestProperty("contentType", "UTF-8");
return connection;
}
// 2.assign every task start index and end index out of the file
private void assignTasks() throws IOException {
for (int i = 0; i < taskNum; i++) {
int size = fileSize / taskNum;
int startIndex = i * size;
int endIndex = i == taskNum - 1 ? fileSize - 1 : i * size + size - 1;
this.startIndexes[i] = startIndex;
this.endIndexes[i] = endIndex;
}
}
// 3.create the local file and temp directory
private void createFiles() throws IOException {
storeDirFile = new File(storeDir);
storeDirFile.mkdirs();
localFile = new File(storeDirFile, fileName);
dtDirFile = new File(storeDirFile, "." + jobId);
dtDirFile.mkdirs();
if (!localFile.exists()) {
RandomAccessFile raf = new RandomAccessFile(localFile, "rw");
raf.setLength(fileSize);
raf.close();
}
}
// 4.let the task start to do their work
private void startTasks() throws IOException {
for (int i = 0; i < taskNum; i++) {
Task task = new Task(this, i);
tasks[i] = task;
task.start();
}
}
private int totalReadBytes() {
int totalReadBytes = 0;
for (int i = 0; i < progress.length; i++) {
totalReadBytes += progress[i];
}
return totalReadBytes;
}
public int[] getStartIndexes() {
return startIndexes;
}
public int[] getEndIndexes() {
return endIndexes;
}
public void writeLocalFile(int startIndex, byte[] buf, int off, int len) throws IOException {
if (rafLocalTl.get() == null) {
RandomAccessFile raf = new RandomAccessFile(localFile, "rw");
rafLocalTl.set(raf);
}
RandomAccessFile raf = rafLocalTl.get();
raf.seek(startIndex);
raf.write(buf, off, len);
}
// 5.let task to report their progress
public void reportProgress(int index, int readBytes) {
progress[index] = readBytes;
}
public void closeTaskResource(int index) throws IOException {
RandomAccessFile raf = rafLocalTl.get();
if (raf != null) {
raf.close();
}
raf = rafOffsetTl.get();
if (raf != null) {
raf.close();
}
}
public void commitOffset(int index, int offset) throws IOException {
File offsetFile = new File(dtDirFile, String.valueOf(index));
if (rafOffsetTl.get() == null) {
RandomAccessFile raf = new RandomAccessFile(offsetFile, "rw");
rafOffsetTl.set(raf);
}
RandomAccessFile raf = rafOffsetTl.get();
raf.seek(0);
raf.writeInt(offset);
}
public int readOffset(int index) throws IOException {
File offsetFile = new File(dtDirFile, String.valueOf(index));
if (offsetFile.exists()) {
RandomAccessFile raf = new RandomAccessFile(offsetFile, "rw");
raf.seek(0);
int offset = raf.readInt();
raf.close();
return offset;
}
return 0;
}
public void reStartTask(int index) throws IOException {
Task task = new Task(this, index);
tasks[index] = task;
task.start();
System.out.println("任务" + index + "发生错误，重新调度该任务");
}
public void taskFinished() {
latch.countDown();
}
private class ProgressThread extends Thread {
private DecimalFormat decimalFormat = new DecimalFormat();
public void run() {
decimalFormat.applyPattern("0.0");
while (true) {
try {
int endPointX = totalReadBytes();
TimeUnit.SECONDS.sleep(1);
int endPointY = totalReadBytes();
int waitSeconds = 1;
while (endPointY - endPointX == 0) {
TimeUnit.SECONDS.sleep(1);
waitSeconds++;
endPointY = totalReadBytes();
}
int speed = (endPointY - endPointX) / waitSeconds;
String speedStr = speed > 1024 ? speed/1024+"kb/s":speed+"b/s";
String percent = decimalFormat.format(endPointY * 100.0 / fileSize);
int remainSeconds = (fileSize - endPointY)/speed;
System.out.println("下载完成"+percent+"%,速度"+speedStr+",估计还需要"+remainSeconds+"秒");
if("100.0".equals(percent)) {
break;
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
}

package org.blackfoxer.cat;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
public class Task extends Thread {
private Job owner;
private int index;
private int readBytes;
private int startIndex;
private int endIndex;
public Task(Job owner,int index) throws IOException {
this.owner = owner;
this.index = index;
if(owner.readOffset(index)!=0) {
this.readBytes = owner.readOffset(index)-owner.getStartIndexes()[index];
owner.reportProgress(index, readBytes);
}
this.startIndex = owner.getStartIndexes()[index]+readBytes;
this.endIndex = owner.getEndIndexes()[index];
}
public void run() {
InputStream inputStream = null;
HttpURLConnection connection = null;
try {
if(startIndex > endIndex) {
owner.taskFinished();
return;
}
connection = owner.createConnection();
connection.setRequestMethod("GET");
String range = "bytes="+startIndex+"-"+endIndex ;
connection.setRequestProperty("Range", range);
if(connection.getResponseCode()==206) {
inputStream = connection.getInputStream();
int len = -1;
byte buf[] = new byte[1024];
int offset = startIndex;
while((len=inputStream.read(buf))!=-1) {
owner.writeLocalFile(offset,buf,0,len);
readBytes+=len;
offset+=len;
owner.commitOffset(index,offset);
owner.reportProgress(index,readBytes);
}
owner.taskFinished();
}
} catch (IOException e) {
e.printStackTrace();
try {
owner.reStartTask(index);
} catch (IOException e1) {
e1.printStackTrace();
}
} finally {
if(inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(connection != null) {
connection.disconnect();
}
try {
owner.closeTaskResource(index);
} catch (IOException e) {
e.printStackTrace();
}
}
}
}

package org.blackfoxer.cat;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
public class JavaXunlei {
private static final BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
private static String storeDir = null;
public static void main(String args[]) throws Exception {
storeDir = getInput("请先设置你的文件存储目录:");
int taskNum = getIntInput("请输入你的开启下载的线程数:");
while(true) {
String url = getInput("请输入文件链接地址:");
Job job = new Job(url,storeDir,taskNum);
job.startJob();
}
}
private static int getIntInput(String message) throws IOException {
String number = getInput(message);
while(!number.matches("\\d+")) {
System.out.println("线程数必须是1个整数");
number = getInput(message);
}
return Integer.parseInt(number);
}
private static String getInput(String message) throws IOException {
System.out.print(message);
String line = in.readLine();
while(line == null || line.trim().length()<1) {
System.out.print(message);
line = in.readLine();
}
return line.trim();
}
}