hbase

最新推荐文章于 2025-05-02 21:59:37 发布

原创最新推荐文章于 2025-05-02 21:59:37 发布 · 232 阅读

CC 4.0 BY-SA版权

public class a  extends Mapper<LongWritable , Text,NullWritable,Text>{
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String s = value.toString();
        if (s.isEmpty()){
            return;
        }
        String[] split = s.split(":");
        if (split.length>10){
            String s4 = "";
            String s1 = s.substring(s.indexOf(split[9])).replace(":", "&");
            String s2 = s.substring(0, s.indexOf(split[9]));
            String s3 = s2.replace(" & ", ",");
            s4 = s3+s1;
            context.write(NullWritable.get(),new Text(s4));
        }

    }
}

public class DSReduce extends Reducer<NullWritable,Text,Text,Text> {
    @Override
    protected void reduce(NullWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        FileSystem fs = null;

        try {
            fs = FileSystem.get(new URI("hdfs://192.168.100.191:8020"), new Configuration());
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }

        FSDataOutputStream fsDataOutputStream = fs.create(new Path("/video.txt"));

        for (Text value : values) {

            fsDataOutputStream.write((value.toString()+"\r\n" +
                    "").getBytes());
        }
    }
}

public class DSDriver extends Configured implements Tool {
    @Override
    public int run(String[] args) throws Exception {

        Configuration _conf = new Configuration();

        Job job = new Job(_conf, "DataSplit");
        job.setJarByClass(DSDriver.class);

        //设置map输出类型
        job.setMapperClass(DSMapper.class);
        job.setMapOutputKeyClass(NullWritable.class);
        job.setMapOutputValueClass(Text.class);

        //设置reduce输出类型
        job.setReducerClass(DSReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        //设置输入
        job.setInputFormatClass(TextInputFormat.class);
        // TextInputFormat.setInputPaths(job,new Path(args[0]));
        TextInputFormat.setInputPaths(job,new Path("F:\\wordcount\\video.txt"));

        //设置输出
        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job,new Path("hdfs://192.168.100.191:8020//DS"));

        return job.waitForCompletion(true)?0:1;
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        ToolRunner.run(configuration,new DSDriver(),args);
    }
}