慢慢读,java断点续传多线程下载

java下载断点续传简单而言就是记录上次下载的位置,下次直接跳到这个位置开始下载就好了。

断点继续下载算法

package com.javaer.examples.file;
 
 
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
 
/**
 * 
 * @author why
 *
 */
public class MutiGoOn {
	//文件目录、文件名
	public String fileDir = "E:/MyDownLoad";
	public String fileName;
	//超时重连时间
	public long reconnectTime = 5;
	//线程数
	private int poolSize = 5;
	//每个线程的缓冲区大小
	public int bufferSize = 1024;
	//url地址
	private String urlLocation = null;
 
	public MutiGoOn(){}
	public MutiGoOn(String url){
		this.urlLocation = url;
	}
	public void downLoad(){
		if(this.urlLocation == null || "".equals(this.urlLocation))return;
		downLoad(this.urlLocation);
	}
	public void downLoad(String urlLocation){
		File file = null;
		File tempFile = null;
		CountDownLatch latch;
		URL url = null;
		ExecutorService pool = Executors.newCachedThreadPool();
		long contentLength = 0;
		long threadLength = 0;
		try {
			//如果未指定名称,则从url中获得下载的文件格式与名字
			if(fileName == null || "".equals(fileName)){
				this.fileName = urlLocation.substring(urlLocation.lastIndexOf("/") + 1,
						urlLocation.lastIndexOf("?") > 0 ? urlLocation.lastIndexOf("?")
								: urlLocation.length());
				if ("".equalsIgnoreCase(this.fileName)) {
					this.fileName = UUID.randomUUID().toString();
				}
			}
			new File(fileDir).mkdirs();
			file = new File(fileDir + File.separator + fileName);
			tempFile = new File(fileDir + File.separator + fileName + "_temp");
 
			url = new URL(urlLocation);
			HttpURLConnection conn = (HttpURLConnection) url.openConnection();
			setHeader(conn);
			//得到content的长度
			contentLength = conn.getContentLength();
 
			System.out.println("total length=" + contentLength);
 
			//把context分为poolSize段,计算每段的长度。
//			threadLength = contentLength / this.poolSize;
			BigDecimal b1 = new BigDecimal(Double.toString(contentLength));
			BigDecimal b2 = new BigDecimal(Double.toString(this.poolSize));
			threadLength = b1.divide(b2, 0, BigDecimal.ROUND_HALF_UP).longValue();
 
			if(file.exists() && tempFile.exists()){
				//如果文件已存在,根据临时文件中记载的线程数量,继续上次的任务
				latch = new CountDownLatch((int)tempFile.length()/28);
				for(int i=0;i<tempFile.length()/28;i++){
					pool.submit(new MutiGoOnTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));
				}
			}else{
				//如果下载的目标文件不存在,则创建新文件
				latch = new CountDownLatch(poolSize);
				file.createNewFile();
				tempFile.createNewFile();
				DataOutputStream os = new DataOutputStream(new FileOutputStream(tempFile));
				for(int i=0;i<this.poolSize;i++){
					os.writeInt(i+1);
					os.writeLong(i*threadLength);
					if(i==this.poolSize-1){//最后一个线程的结束位置应为文件末端
						os.writeLong(contentLength);
					}else{
						os.writeLong((i+1)*threadLength);
					}
					os.writeLong(i*threadLength);
					pool.submit(new MutiGoOnTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));
				}
				os.close();
			}
			//等待下载任务完成
			latch.await();
			//删除临时文件
			tempFile.delete();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		} finally{
			pool.shutdown(); 
		}
	}
 
	private void setHeader(URLConnection conn) {
		conn.setRequestProperty("User-Agent",
						"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
		conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");
		conn.setRequestProperty("Accept-Encoding", "aa");
		conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
		conn.setRequestProperty("Keep-Alive", "300");
		conn.setRequestProperty("Connection", "keep-alive");
		conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");
		conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");
		conn.setRequestProperty("Cache-Control", "max-age=0");
		conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");
	}
 
 
	public String getFileDir() {
		return fileDir;
	}
	public void setFileDir(String fileDir) {
		this.fileDir = fileDir;
	}
	public String getFileName() {
		return fileName;
	}
	public void setFileName(String fileName) {
		this.fileName = fileName;
	}
	public long getReconnectTime() {
		return reconnectTime;
	}
	public void setReconnectTime(long reconnectTime) {
		this.reconnectTime = reconnectTime;
	}
	public int getPoolSize() {
		return poolSize;
	}
	public void setPoolSize(int poolSize) {
		this.poolSize = poolSize;
	}
	public int getBufferSize() {
		return bufferSize;
	}
	public void setBufferSize(int bufferSize) {
		this.bufferSize = bufferSize;
	}
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		MutiGoOn dl = new MutiGoOn();
		dl.setFileDir("/");
		dl.setFileName("openx.rar");
		dl.setPoolSize(20);
		long beginTime = System.currentTimeMillis();
		dl.downLoad("http://s2.knowsky.com/code/php/openx-220080603.rar");
		long endTime = System.currentTimeMillis();
		BigDecimal b1 = new BigDecimal(endTime - beginTime);
		BigDecimal b2 = new BigDecimal(1000);
		double cost = b1.divide(b2, 2, BigDecimal.ROUND_HALF_UP).doubleValue();
		System.out.println("Time cost:" + cost + "s");
	}
 
}

下载算法

package com.javaer.examples.file;
 
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
 
/**
 * 
 * @author why
 *
 */
public class MutiGoOnTask implements Callable<String>{
	//超时重连时间
	private long reconnectTime = 5;
	//缓冲区大小
	private int bufferSize = 1024;
 
	private CountDownLatch latch;
	private RandomAccessFile file = null;
	private RandomAccessFile tempFile = null;
	private URL url = null;
	private int id;
	private long startPosition;
	private long endPosition;
	private long currentPosition ;
 
	public MutiGoOnTask(File file,File tempFile,URL url,int id,CountDownLatch latch,long reconnectTime,int bufferSize){
		try {
			this.file = new RandomAccessFile(file, "rw");
			this.tempFile = new RandomAccessFile(tempFile, "rw");
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
		this.url = url;
		this.id = id;
		this.latch = latch;
	}
 
	public String call(){
 
		try {
			tempFile.seek((id-1)*28);
			tempFile.readInt();
			this.startPosition = tempFile.readLong();
			this.endPosition = tempFile.readLong();
			this.currentPosition = tempFile.readLong();
		} catch (IOException e) {
			e.printStackTrace();
		}
 
		System.out.println("Thread " + id + " begin!");
 
		HttpURLConnection conn = null;
		InputStream inputStream = null;
 
		while(true){
			try {
				tempFile.seek(id*28 - 8);
				// 打开URLConnection
				conn = (HttpURLConnection) this.url.openConnection();
				setHeader(conn);
				// 设置连接超时时间为10000ms
				conn.setConnectTimeout(10000);
				// 设置读取数据超时时间为10000ms
				conn.setReadTimeout(10000);
 
				if (currentPosition < endPosition) {
					// 设置下载数据的起止区间
					conn.setRequestProperty("Range", "bytes=" + currentPosition + "-" + endPosition);
 
					System.out.println("Thread " + id + " startPosition=" + startPosition 
							+ ",endPosition=" + endPosition + ",currentPosition=" + currentPosition);
 
					file.seek(currentPosition);
 
					// 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK
					// 如果不是以上两种状态,把status改为STATUS_HTTPSTATUS_ERROR
					if (conn.getResponseCode() != HttpURLConnection.HTTP_OK
							&& conn.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) {
						System.out.println("Thread " + id + ": code = " + conn.getResponseCode() + ", status = " + conn.getResponseMessage());
						file.close();
						conn.disconnect();
						System.out.println("Thread " + id + " finished.");
						break;
					}
 
					inputStream = conn.getInputStream();
					int len = 0;
					byte[] b = new byte[bufferSize];
					while ((len = inputStream.read(b)) != -1) {
						file.write(b, 0, len);
 
						currentPosition += len;
						// set tempFile now position
						tempFile.seek(id*28 - 8);
						tempFile.writeLong(currentPosition);
					}
 
					file.close();
					tempFile.close();
					inputStream.close();
					conn.disconnect();
				}
 
				System.out.println("Thread " + id + " finished.");
				break;
			} catch (IOException e) {
				try {
					TimeUnit.SECONDS.sleep(getReconnectTime());
				} catch (InterruptedException e1) {
					e1.printStackTrace();
				}
				continue;
			}
		}
		latch.countDown();
		return "finish";
	}
 
	private void setHeader(URLConnection conn) {
		conn.setRequestProperty("User-Agent",
						"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
		conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");
		conn.setRequestProperty("Accept-Encoding", "aa");
		conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
		conn.setRequestProperty("Keep-Alive", "300");
		conn.setRequestProperty("Connection", "keep-alive");
		conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");
		conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");
		conn.setRequestProperty("Cache-Control", "max-age=0");
		conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");
	}
 
	public long getReconnectTime() {
		return reconnectTime;
	}
 
	public void setReconnectTime(long reconnectTime) {
		this.reconnectTime = reconnectTime;
	}
 
	public int getBufferSize() {
		return bufferSize;
	}
 
	public void setBufferSize(int bufferSize) {
		this.bufferSize = bufferSize;
	}
 
 
}


This entry was posted in JAVA and tagged , , . Bookmark the permalink.
月小升QQ 2651044202, 技术交流QQ群 178491360
首发地址:月小升博客https://java-er.com/blog/java-d-dow/
无特殊说明,文章均为月小升原创,欢迎转载,转载请注明本文地址,谢谢
您的评论是我写作的动力

2 Responses to 慢慢读,java断点续传多线程下载

  1. 益**汤 says:

    佩服,佩服,可以转载吗?

Leave a Reply

Your email address will not be published. Required fields are marked *

*

*

  

  

About Me

静水流深,水滴石穿