outputformat自定义--数据过滤

最新推荐文章于 2023-01-10 09:24:24 发布

领悟大数据

最新推荐文章于 2023-01-10 09:24:24 发布

阅读量312

点赞数 1

CC 4.0 BY-SA版权

分类专栏： hadoop 文章标签： outputformat 数据过滤

本文链接：https://blog.youkuaiyun.com/weixin_42898914/article/details/84891366

hadoop 专栏收录该内容

22 篇文章

订阅专栏

本文介绍了一种使用自定义OutputFormat进行日志文件过滤的方法，通过MapReduce作业将包含特定关键字的日志分离到不同的文件中。该技术适用于大规模日志数据处理场景。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

outputformat自定义--数据过滤
   需求：过滤日志文件
       把包含itstaredu的放在一个文件
       把不包含itstaredu的放在一个文件
       public class FuncFileOutputFormat extends FileOutputFormat<Text, NullWritable>{
           @Override
           public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttempContext job)
                   throws IOException, InterruptException {
               FileRecordWriter fileRecordWriter = new FileRecordWriter(job);

               return fileRecordWriter;
           }
       }

       public class FileRecordWriter extends RecordWriter<Text, NullWritable>{
           FSDataOutputStream other = null;
           FSDataOutputStream itstarlog = null;
           Configuration conf = null;

           //1、定义数据输出路径
           public FileRecordWriter(TaskAttempContext job){
               //需要配置信息
               conf = job.getConfiguration();

//获取文件系统
FileSystem fs = FileSystem.get(conf);

               //定义输出路径
               itstarlog = fs.create(new Path("C:/outitstaredu/itstar.logs"));
               other = fs.create(new Path("C:/outputother/other.logs"));

           }
           //2、数据输出
           @Override
           public void writer(Text key, NullWritable value) throws IOException, InterruptException{
               //判断的话根据key
               if(key.toString().contains("itstar")){
                   //写出道文件
                   itstarlog.write(key.getBytes());
               }else if {
                   other.write(key.getBytes());
               }

           }
           //3、关闭资源
           @Override
           public void close(TaskAttempContext context) throws IOException, InterruptException{
               if(null != itstarlog){
                   itstarlog.close();
               }
               if(null != other){
                   other.close();
               }

           }
       }
       public class FileMapper extends Mapper<LongWritable, Text, Text, NullWritable>{

           @Override
           public void map(LongWritable key, Text value, Context context)
                   throw IOException, InterruptException{
               //输出
               context.write(value, NullWritable.get());

}
}

       public class FileReducer extends Reducer<Text, NullWritable, Text, NullWritable>{

           @Override
           public void reduce(Text key, Interable<NullWritable> value, Context context)
                   throw IOException, InterruptException{
               //输出
               String k = key.toString;

context.write(new Text(k), NullWritable.get());

}
}

       public class FileDriver{
       public static void main(String[] args) throws IOException, ClassNotFoundException,InterruptException{
           Configuration conf = new Configuration();
           Job job = Job.getInstance();

job.setJarByClass(FileDriver.class);

job.setMapperClass(FileMapper.class);
job.setReducerClass(FileReducer.class);

job.setMapOutputValueClass(NullWritable.class);
job.setMapOutputKeyClass(Text.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);

//设置自定义的outputformat
job.setOutputFormatClass(FuncFileOutputFormat.class);

FileInputFormat.setInputPaths(job, new Path("C:/in"));
FileOutputFormat.setOutputPath(job, new Path("c:/out"));

           job.waitForCompletion(true);
       }
   }