基于Hadoop的数据云盘的实现

环境：centos6.6   Tomcat7.0  eclipse  MySQL  

首先介绍一下这个简单网盘的功能：实现用户的注册，登录；上传、下载、删除文件；

基于Hadoop hdfs的集群分布式系统做成的这个简单的云盘实现；

首先启动centos，打开安装好的eclipse，file-->New-->Dynamic Application Project

注：如果没有这个项目，安装Java EE 

   

项目名称自己设置，一直点击下一步，最后一步时，最后一步最好勾选创建web.xml，没有勾选后面自己在WEB-INF下new一个名为web.xml的xml File

           创建后如图 

创建后，导入Hadoop下的jar包，位于安装Hadoop文件下下的share文件中，选中后粘贴到WEB-INF下的lib文件中

eclipse中选中所有的jar包，右键-->Build Path-->Add to build path。或者是选中项目，按alt+enter,进入界面，左侧选择java build path ,右侧libraries, add jars,

选择你要导入的jar包ok就好啦，

接下来就是真正开始创建了，选中java Resources，右键new-->package,创建三个package,可自己命名，我这里分别是，com.Bean, com.model, com.controller

右键com.controller,创建上传文件的名为uploadServlet.java类文件

package com.controller;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;

import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.hadoop.fs.FileStatus;

import com.simple.model.HdfsDAO;

public class UploadServlet extends HttpServlet{
	protected void doGet(HttpServletRequest request, HttpServletResponse response)throws ServletException, IOException{
		this.doPost(request,response);
	}
	protected void doPost(HttpServletRequest request, HttpServletResponse response)throws ServletException, IOException{
		request.setCharacterEncoding("utf-8");
		File file;
		int maxFileSize = 50 * 1024 * 1024;  //50M
		int maxMemSize = 50 * 1024 * 1024;  //50M
		ServletContext context = getServletContext();
		String filePath = context.getInitParameter("file-upload");
		System.out.println("source file path:"+filePath+"");
		//验证上传内容的类型
		String contentType = request.getContentType();
		if((contentType.indexOf("multipart/form-data") >= 0)){
			DiskFileItemFactory factory = new DiskFileItemFactory();
			//设置内存中存储文件的最大值
			factory.setSizeThreshold(maxMemSize);
			//本地存储的数据大于maxMemSize
			factory.setRepository(new File("/soft/file-directory"));
			//创建一个新的文件上传处理程序
			ServletFileUpload upload = new ServletFileUpload(factory);
			//设置最大上传文件大小
			upload.setSizeMax(maxFileSize);
			try{
				//解析获取新文件
				List fileItems = upload.parseRequest(request);
				//处理上传的文件
				Iterator i = fileItems.iterator();
				System.out.println("begin to upload file to tomcat server</p>");
				while(i.hasNext()){
					FileItem fi = (FileItem)i.next();
					if(!fi.isFormField()){
						//获取上传文件的参数
						String fieldName = fi.getFieldName();
						String fileName = fi.getName();
						String fn = fileName.substring(fileName.lastIndexOf("/")+1);
						System.out.println("<br>"+fn+"<br>");
						boolean isInMemory = fi.isInMemory();
						long sizeInBytes = fi.getSize();
						//写入文件
						if(fileName.lastIndexOf("/") >= 0 ){
							file = new File(filePath,fileName.substring(fileName.lastIndexOf("/")));
						}else{
							file = new File(filePath,fileName.substring(fileName.lastIndexOf("/")+1));
						}
						fi.write(file);
						System.out.println("upload file to tomcat server success!");
						System.out.println("begin to upload file to hadoop hdfs</p>");
						String name = filePath + "/"+fileName;
						//将tomcat上的文件上传到hadoop上
						HdfsDAO hdfs = new HdfsDAO();
						hdfs.copyFile(name);
						System.out.println("upload file to hadoop hdfs success!");
						FileStatus[] documentList = hdfs.getDirectoryFromHdfs();
						request.setAttribute("documentList", documentList);
						System.out.println("得到list数据"+documentList);
						request.getRequestDispatcher("index.jsp").forward(request, response);
					}
				}
			}catch(Exception ex){
				System.out.println(ex);
			}
		}else{
			System.out.println("<p>No file uploaded</p>");
		}
	}
}

继续在com.controller下创建名为downloadServlet.java类文件，

package com.controller;

import java.io.IOException;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.hadoop.fs.FileStatus;

import com.simple.model.HdfsDAO;

public class DownloadServlet extends HttpServlet{
	private static final long serialVersionUID = 1L;
	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException{
		String local = "/soft/download-directory";//选择一个你centos下的目录作为你的下载文件存放地址
		String filePath = new String(request.getParameter("filePath").getBytes("ISO-8859-1"),"GB2312");
		HdfsDAO hdfs = new HdfsDAO();
		hdfs.download(filePath, local);
		FileStatus[] documentList = hdfs.getDirectoryFromHdfs();
		request.setAttribute("documentList", documentList);
		System.out.println("得到list数据"+documentList);
		request.getRequestDispatcher("index.jsp").forward(request, response);
	}
	protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException{
		this.d