hadoop学习笔记 day02 hadoop Java环境的API

package day1;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.util.HashMap;
import java.util.Map;

public class HDFSdemo {
    FileSystem fs = null;
    //客户端Java的API
    //获得客户端对象
    @Before
    public void init() throws IOException {
        System.setProperty("HADOOP_USER_NAME","root");

        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://pc-01:9000");//不配置默认本机
        fs = FileSystem.get(conf);
    }
    //创建目录
    @Test
    public void testMkdirs() throws IOException {

        fs.mkdirs(new Path("/bbb/zzz/"));
        fs.close();
    }
    //删除文件
    @Test
    public void testDelete() throws IOException {

        fs.delete(new Path("/bbb/zzz/"),true);
        fs.close();
    }

    //移动 重命名
    @Test
    public void testMv() throws IOException {

        fs.rename(new Path("/bbb/zxz/"),new Path("/zxz/")); //移动和重命名是一回事
        fs.close();
    }
    //上传文件
    @Test
    public void testPut() throws IOException {

        fs.copyFromLocalFile(new Path("d:/a.jpg"),new Path("/zxz/"));
        fs.close();
    }
    //下载文件
    @Test
    public void testDownload() throws IOException {

        fs.copyToLocalFile(new Path("/zxz/"),new Path("d:/java"));
        fs.close();
    }
    //递归查看目录内容
    @Test
    public void testLs1() throws IOException {

        RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
        while (files.hasNext()) {
            LocatedFileStatus next = files.next();
            System.out.println(next);
        }
        fs.close();
    }
    //查看当前目录内容
    @Test
    public void testLs2() throws IOException {

        FileStatus[] status = fs.listStatus(new Path("/"));
        for (FileStatus fileStatus : status) {
            System.out.println(fileStatus.toString());
        }
        fs.close();
    }
    //读取文件内容    (wordcount)
    @Test
    public void testReaddata() throws IOException {
        HashMap<String, Integer> cotMap = new HashMap<>();
        FSDataInputStream in = fs.open(new Path("/aaa1.txt"));
        BufferedReader br = new BufferedReader(new InputStreamReader(in));
        String next;
        while ((next = br.readLine()) != null) {
            System.out.println(next);
            String[] split = next.split(" "); //按照空格分割单词
            for (String s : split) {
                cotMap.put(s,cotMap.getOrDefault(s,0)+1); //单词和出现次数存入map
            }
        }
        for (Map.Entry<String, Integer> entry : cotMap.entrySet()) {
            System.out.println(entry.getKey()+"--->"+entry.getValue());
        }

        in.close();

		//将计数结果重新存入HDFS中
        FSDataOutputStream append = fs.create(new Path("/result.txt"));
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(append));
        for (Map.Entry<String, Integer> entry : cotMap.entrySet()) {
            bw.write(entry.getKey()+"--->"+entry.getValue());
            bw.newLine();
        }
        bw.flush();
        bw.close();
        fs.close();
    }

    //向文件中追加内容
    @Test
    public void testAppendData() throws IOException {
        FSDataOutputStream os = fs.append(new Path("/aaa1.txt"));
        os.write("这个男的已经21K了".getBytes());
        os.close();
        fs.close();
    }
}