头哥 | 大数据系统及应用-HDFS实训

若未出现预期结果可私信我答疑
我是头歌闯关王涉猎头歌7千多关,如有其他关卡也可私信我

2025年3月30日

第1关：HDFS Java API编程 ——文件读写

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class hdfs {
    public static void main(String[] args) throws IOException {
        //请在 Begin-End 之间添加代码，完成任务要求。
        /********* Begin *********/
        
        // 1. 获取Hadoop系统配置
        Configuration conf = new Configuration();
        
        // 2. 获取HDFS文件系统实例
        FileSystem fs = FileSystem.get(conf);
        
        // 3. 创建文件路径
        Path filePath = new Path("/user/hadoop/myfile");
        
        // 4. 创建文件并写入内容
        FSDataOutputStream out = fs.create(filePath);
        out.writeUTF("https://www.educoder.net");
        out.close();
        
        // 5. 读取文件内容
        FSDataInputStream in = fs.open(filePath);
        String content = in.readUTF();
        in.close();
        
        // 6. 输出文件内容
        System.out.println(content);
        
        // 7. 关闭文件系统连接
        fs.close();
        
        /********* End *********/
    }
}

第2关：HDFS Java API编程——文件上传

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;

public class hdfs {

    /**
     * 判断路径是否存在
     */
    public static boolean test(Configuration conf, String path) throws IOException {
        /*****start*****/
        FileSystem fs = FileSystem.get(conf);
        Path filePath = new Path(path);
        boolean exists = fs.exists(filePath);
        fs.close();
        return exists;
        /*****end*****/
    }

    /**
     * 复制文件到指定路径
     * 若路径已存在，则进行覆盖
     */
    public static void copyFromLocalFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException {
        /*****start*****/
        FileSystem fs = FileSystem.get(conf);
        Path localPath = new Path(localFilePath);
        Path remotePath = new Path(remoteFilePath);
        fs.copyFromLocalFile(false, true, localPath, remotePath);
        fs.close();
        /*****end*****/
    }
 
    /**
     * 追加文件内容
     */
    public static void appendToFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException {
       /*****start*****/
        FileSystem fs = FileSystem.get(conf);
        Path remotePath = new Path(remoteFilePath);
        FSDataOutputStream out = fs.append(remotePath);
        BufferedReader br = new BufferedReader(new FileReader(localFilePath));
        String line;
        while ((line = br.readLine()) != null) {
            out.writeBytes(line + "\n");
        }
        br.close();
        out.close();
        fs.close();
        /*****end*****/
    }
    
    /**
     * 主函数
     */
    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        conf.set("dfs.support.append", "true");
        conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
        conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
  
        createHDFSFile(conf);
     
        String localFilePath = "./file/text.txt";         // 本地路径
        String remoteFilePath = "/user/hadoop/text.txt"; // HDFS路径
        String choice = "";    
        
        try {
            /* 判断文件是否存在 */
            Boolean fileExists = false;
            if (hdfs.test(conf, remoteFilePath)) {
                fileExists = true;
                System.out.println(remoteFilePath + " 已存在.");
                choice = "append";     //若文件存在则追加到文件末尾
            } else {
                System.out.println(remoteFilePath + " 不存在.");
                choice = "overwrite";    //覆盖
            }
            
            /*****start*****/
            if (!fileExists) { // 文件不存在，则上传
                copyFromLocalFile(conf, localFilePath, remoteFilePath);
                System.out.println(localFilePath + " 已上传至 " + remoteFilePath);
            } else if (choice.equals("overwrite")) {    // 选择覆盖
                copyFromLocalFile(conf, localFilePath, remoteFilePath);
                System.out.println(localFilePath + " 已覆盖 " + remoteFilePath);
            } else if (choice.equals("append")) {   // 选择追加
                appendToFile(conf, localFilePath, remoteFilePath);
                System.out.println(localFilePath + " 已追加至 " + remoteFilePath);
            }
            /*****end*****/
            
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //创建HDFS文件
    public static void createHDFSFile(Configuration conf) throws IOException {
        FileSystem fs = FileSystem.get(conf);  //获取文件系统
        Path file = new Path("/user/hadoop/text.txt");        //创建文件   
        FSDataOutputStream outStream = fs.create(file); //获取输出流
        outStream.writeUTF("hello");
        outStream.close();
        fs.close();
    }
}

第3关：HDFS Java API编程 ——文件下载

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;

public class hdfs {
    /**
     * 下载文件到本地
     * 判断本地路径是否已存在，若已存在，则自动进行重命名
     */
    public static void copyToLocal(Configuration conf, String remoteFilePath, String localFilePath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path remotePath = new Path(remoteFilePath);
        File f = new File(localFilePath);
        
        /*****start*****/
        /*在此添加判断文件是否存在的代码，如果文件名存在，自动重命名(在文件名后面加上 _0, _1 ...) */
        if (f.exists()) {
            System.out.println(localFilePath + " 已存在.");
            Integer i = 0;
            String baseName = localFilePath;
            while (true) {
                String newName = baseName + "_" + i;
                f = new File(newName);
                if (!f.exists()) {
                    localFilePath = newName;
                    break;
                }
                i++;
            }
            System.out.println("将重新命名为: " + localFilePath);
        }
        /*****end*****/

        /*****start*****/
        // 在此添加将文件下载到本地的代码
        Path localPath = new Path(localFilePath);
        fs.copyToLocalFile(false, remotePath, localPath);
        System.out.println("文件已下载到: " + localFilePath);
        /*****end*****/
        
        fs.close();
    }
    
    /**
     * 主函数
     */
    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        createHDFSFile(conf);
        String localFilePath = "/tmp/output/text.txt";    // 本地路径
        String remoteFilePath = "/user/hadoop/text.txt";    // HDFS路径
        
        try {
            //调用方法下载文件至本地
            hdfs.copyToLocal(conf, remoteFilePath, localFilePath);
            System.out.println("下载完成");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //创建HDFS文件
    public static void createHDFSFile(Configuration conf) throws IOException {
        FileSystem fs = FileSystem.get(conf);  //获取文件系统
        Path file = new Path("/user/hadoop/text.txt");        //创建文件   
        FSDataOutputStream outStream = fs.create(file); //获取输出流
        outStream.writeUTF("hello hadoop HDFS www.educoder.net");
        outStream.close();
        fs.close();
    }
}

第4关：HDFS Java API编程 ——使用字符流读取数据

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;

public class hdfs {
    /**
     * 读取文件内容
     */
    public static void cat(Configuration conf, String remoteFilePath) throws IOException {
        /*****start*****/
        //1.读取文件中的数据
        FileSystem fs = FileSystem.get(conf);
        Path path = new Path(remoteFilePath);
        FSDataInputStream in = fs.open(path);
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        
        //2.将读取到的数据输出到/tmp/output/text.txt文件中
        File outputFile = new File("/tmp/output/text.txt");
        outputFile.getParentFile().mkdirs(); // 确保目录存在
        FileWriter writer = new FileWriter(outputFile);
        
        String line;
        while ((line = reader.readLine()) != null) {
            writer.write(line + "\n");
        }
        
        // 关闭资源
        reader.close();
        in.close();
        writer.close();
        fs.close();
        /*****end*****/
    }
    
    /**
     * 主函数
     */
    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        createHDFSFile(conf);
        String remoteFilePath = "/user/hadoop/text.txt";    // HDFS路径
        
        try {
            System.out.println("读取文件: " + remoteFilePath);
            hdfs.cat(conf, remoteFilePath);
            System.out.println("\n读取完成");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //创建HDFS文件
    public static void createHDFSFile(Configuration conf) throws IOException {
        FileSystem fs = FileSystem.get(conf);  //获取文件系统
        Path file = new Path("/user/hadoop/text.txt");        //创建文件   
        FSDataOutputStream outStream = fs.create(file); //获取输出流
        outStream.writeUTF("hello hadoop HDFS step4 www.educoder.net");
        outStream.close();
        fs.close();
    }
}

第5关：HDFS Java API编程 ——删除文件

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;

public class hdfs {

    /**
     * 删除文件 
     */
    public static boolean rm(Configuration conf, String remoteFilePath) throws IOException {
        /*****start*****/        
        // 获取文件系统
        FileSystem fs = FileSystem.get(conf);
        // 创建路径对象
        Path path = new Path(remoteFilePath);
        // 删除文件，返回操作结果
        boolean result = fs.delete(path, false);
        fs.close();
        return result;
        /*****end*****/
    }
    
    /**
     * 主函数
     */
    public static void main(String[] args) {
        Configuration conf = new Configuration();
        String remoteFilePath = "/user/hadoop/text.txt";    // HDFS文件
        
        try {
            if (rm(conf, remoteFilePath)) {
                System.out.println("文件删除: " + remoteFilePath);
            } else {
                System.out.println("操作失败（文件不存在或删除失败）");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

第6关：HDFS Java API编程 ——删除文件夹

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;

public class hdfs {

    /**
     * 判断目录是否为空
     * true: 空，false: 非空
     */
    public static boolean isDirEmpty(Configuration conf, String remoteDir) throws IOException {
        /*****start*****/
        FileSystem fs = FileSystem.get(conf);
        Path dirPath = new Path(remoteDir);
        // 检查目录是否存在
        if (!fs.exists(dirPath)) {
            fs.close();
            return true;  // 不存在视为空
        }
        // 获取目录内容
        FileStatus[] statuses = fs.listStatus(dirPath);
        fs.close();
        return statuses.length == 0;  // 长度为0表示空目录
        /*****end*****/
    }
    
    /**
     * 删除目录
     */
    public static boolean rmDir(Configuration conf, String remoteDir, boolean recursive) throws IOException {
        /*****start*****/
        FileSystem fs = FileSystem.get(conf);
        Path dirPath = new Path(remoteDir);
        // 检查目录是否存在
        if (!fs.exists(dirPath)) {
            fs.close();
            return false;  // 目录不存在
        }
        // 删除目录
        boolean result = fs.delete(dirPath, recursive);
        fs.close();
        return result;
        /*****end*****/
    }
    
    /**
     * 主函数
     */
    public static void main(String[] args) {
        Configuration conf = new Configuration();
        String remoteDir = "/user/hadoop/dir/";    // HDFS目录
        String remoteDir1 = "/user/hadoop/tmp/";   // HDFS目录
        Boolean forceDelete = false;  // 是否强制删除
        
        try {
            // 处理第一个目录
            if (!isDirEmpty(conf, remoteDir) && !forceDelete) {
                System.out.println("目录 " + remoteDir + " 不为空，不删除");
            } else {
                if (rmDir(conf, remoteDir, forceDelete)) {
                    System.out.println("目录已删除: " + remoteDir);
                } else {
                    System.out.println("操作失败");
                }
            }
            
            // 处理第二个目录
            if (!isDirEmpty(conf, remoteDir1) && !forceDelete) {
                System.out.println("目录 " + remoteDir1 + " 不为空，不删除");
            } else {
                if (rmDir(conf, remoteDir1, forceDelete)) {
                    System.out.println("目录已删除: " + remoteDir1);
                } else {
                    System.out.println("操作失败");
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

第7关：HDFS Java API编程 ——自定义数据输入流

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.*;

public class MyFSDataInputStream extends FSDataInputStream {
    public MyFSDataInputStream(InputStream in) {
        super(in);
    }
    
    /**
     * 实现按行读取
     * 每次读入一个字符，遇到"\n"结束，返回一行内容
     */
    public static String readline(BufferedReader br) throws IOException {
        /*****start*****/
        StringBuilder sb = new StringBuilder();
        int ch;
        while ((ch = br.read()) != -1) {
            if (ch == '\n') {
                break;
            }
            sb.append((char)ch);
        }
        // 如果读取到内容或者遇到换行符但sb不为空，则返回内容
        if (sb.length() > 0 || ch != -1) {
            return sb.toString();
        } else {
            return null;  // 文件末尾返回null
        }
        /*****end*****/
    }

    /**
     * 读取文件内容
     */
    public static void cat(Configuration conf, String remoteFilePath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path remotePath = new Path(remoteFilePath);
        FSDataInputStream in = fs.open(remotePath);
        BufferedReader br = new BufferedReader(new InputStreamReader(in));
        FileWriter f = new FileWriter("/tmp/output/text.txt");
        String line = null;
        while ((line = MyFSDataInputStream.readline(br)) != null) {
            f.write(line + "\n");  // 加上换行符保持原格式
        }
        f.close();
        br.close();
        in.close();
        fs.close();
    }
    
    /**
     * 主函数
     */
    public static void main(String[] args) {
        Configuration conf = new Configuration();
        String remoteFilePath = "/user/hadoop/text.txt";    // HDFS路径
        try {
            MyFSDataInputStream.cat(conf, remoteFilePath);
            System.out.println("文件内容已成功写入/tmp/output/text.txt");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}