- 前提构建好hadoop,并启动hdfs(hadoop)
- 导入相对应的jar包:
本文通过maven打jar包实现需要的包主要为以下三个
https://mvnrepository.com/search?q=hadoop
java编写demo
- 本地上传hdfs
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.FileInputStream;
import java.io.IOException;
public class Uploading {
public static void uploading(String hdfsFile,String hdfsURL,String fileName) throws IOException {
Configuration cfg=new Configuration();
cfg.set("fs.defaultFS",hdfsURL);
FileSystem fs=FileSystem.get(cfg);
if (fs.exists(new Path(hdfsFile))){
FSDataOutputStream fsdos=fs.create(new Path(hdfsFile));
FileInputStream fis=new FileInputStream(fileName);
byte[]bytes=new byte[2048];
int count =fis.read(bytes,0,2048);
while(count>0){
fsdos.write(bytes,0,count);
count=fis.read(bytes,0,2048);
}
fis.close();
fsdos.close();
fs.close();
}
}
public static void main(String[] args) throws IOException {
uploading(args[0],args[1],args[2]);
}
}
- 从hdfs中下载
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.FileOutputStream;
import java.io.IOException;
public class Download {
public static void download(String hdfsFile,String hdfsURL,String fileName) throws IOException {
Configuration cfg=new Configuration();
cfg.set("fs.defaultFS",hdfsURL);
FileSystem fs=FileSystem.get(cfg);
if (fs.exists(new Path(hdfsFile))){
FSDataInputStream fsdis=fs.open(new Path(hdfsFile));
FileOutputStream fos=new FileOutputStream(fileName);
byte[]bytes=new byte[2048];
//首次读
int count =fsdis.read(bytes,0,2048);
while(count>0){
fos.write(bytes,0,count);
count=fsdis.read(bytes,0,2048);
}
fos.close();
fsdis.close();
fs.close();
}
}
public static void main(String[] args) throws IOException {
download(args[0],args[1],args[2]);
}
- 打jar包
- 将jar包导入Linux ,hadoop目录下
- 使用hadoop命令
- 上传
- 下载