学习大数据操作,Hadoop 旗下对hdfs操作入门级的java 编码:
import java.io.InputStream;
import java.net.URL;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
/**
* hdfs简单的操作
* @author hardycheers
*
*/
public class HadoopFirst {
/**
* 读取hdfs上某一文件的内容
* @throws Exception
*/
@Test
public void read() throws Exception {
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
URL url = new URL("hdfs://192.168.163.128:9000/user/hadoop/file5.txt");
InputStream is = url.openStream();
IOUtils.copyBytes(is, System.out, 1024);
IOUtils.closeStream(is);
}
/**
* 展示hdfs目录下的文件列表
* @throws Exception
*/
@Test
public void listFiles() throws Exception {
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
FileSystem fileSystem = FileSystem.get(config);
System.out.println(fileSystem.getClass());
Path path = new Path("/user/hadoop");
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(path, true);
while (listFiles.hasNext()) {
LocatedFileStatus locatedFileStatus = listFiles.next();
System.out.println("路径: "+locatedFileStatus.getPath() + " 大小: "+ locatedFileStatus.getLen());
}
}
/**
* 展示hdfs上的目录结构
* @throws Exception
*/
@Test
public void listStatus() throws Exception {
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
FileSystem fileSystem = FileSystem.get(config);
Path path = new Path("/");
FileStatus[] listStatus = fileSystem.listStatus(path);
for (FileStatus fileStatus : listStatus) {
System.out.println("目录: "+fileStatus.getPath());
}
}
/**
* 创建目录 : 需要增加执行参数:-DHADOOP_USER_NAME=hadoop
* @throws Exception
*/
@Test
public void testMkdir() throws Exception {
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
FileSystem fileSystem = FileSystem.get(config);
Path path = new Path("/bigdata");
if(!fileSystem.exists(path)){
boolean status = fileSystem.mkdirs(path);
System.out.println("创建目录状态: "+status);
}
}
/**
* 删除文件夹操作 需要增加执行参数:-DHADOOP_USER_NAME=hadoop
* @throws Exception
*/
@Test
public void testDeleteFolder() throws Exception {
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
FileSystem fileSystem = FileSystem.get(config);
Path path = new Path("/bigdata");
if(fileSystem.exists(path)){
boolean status = fileSystem.delete(path, true);
System.out.println("删除目录状态: "+status);
}
}
/**
* 上传本地文件到HDFS操作 需要增加执行参数:-DHADOOP_USER_NAME=hadoop
* @throws Exception
*/
@Test
public void uploadFile() throws Exception {
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
FileSystem fileSystem = FileSystem.get(config);
Path path = new Path("/user");
Path source = new Path("D:/test/linuxcmd.txt");
fileSystem.copyFromLocalFile(source,path);
System.out.println("上传完成!!!");
}
/**
* 下载HDFS上的文件到本地操作 需要增加执行参数:-DHADOOP_USER_NAME=hadoop
* @throws Exception
*/
@Test
public void downloadFile() throws Exception {
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
FileSystem fileSystem = FileSystem.get(config);
Path path = new Path("/user/linuxcmd.txt");
Path dist_path = new Path("D:/test/123.txt");
fileSystem.copyToLocalFile(path,dist_path);
System.out.println("下载完成!!!");
}
}
增加执行参数的位置