将hdfs上的数据通过MapReduce上传到hbase上

最新推荐文章于 2023-03-31 13:38:11 发布

wsow

最新推荐文章于 2023-03-31 13:38:11 发布

阅读量364

点赞数

CC 4.0 BY-SA版权

分类专栏： Hbase 文章标签： hbase

本文链接：https://blog.youkuaiyun.com/wsong_14/article/details/77983845

Hbase 专栏收录该内容

10 篇文章

订阅专栏

本文介绍了一个使用HBase进行数据加载的具体实现案例。通过自定义Mapper和Reducer，将原始数据转换为HBase可接受的格式，并创建了名为phone_log的表来存储数据。此过程涉及配置、表结构设计及数据写入等多个环节。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package hbase;  


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.hbase.HBaseConfiguration;  
import org.apache.hadoop.hbase.HColumnDescriptor;  
import org.apache.hadoop.hbase.HTableDescriptor;  
import org.apache.hadoop.hbase.TableName;  
import org.apache.hadoop.hbase.client.HBaseAdmin;  
import org.apache.hadoop.hbase.client.Mutation;  
import org.apache.hadoop.hbase.client.Put;  
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;  
import org.apache.hadoop.hbase.mapreduce.TableReducer;  
import org.apache.hadoop.io.LongWritable;  
import org.apache.hadoop.io.NullWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  


import java.io.IOException;  
import java.text.SimpleDateFormat;  
import java.util.Date;  


/** 
 * Created by Administrator on 2017/3/7. 
 */  
public class LoadData extends Configured {  
    public static class LoadDataMapper extends Mapper<LongWritable,Text,LongWritable,Text>{  
        private Text out = new Text();  
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmss");  

        @Override  
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {  
            //1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC  120.196.100.82  i02.c.aliimg.com        24  27  2481    24681   200
            String line = value.toString();  
            String [] splited = line.split("\t");  
            String  formatedDate = simpleDateFormat.format(new Date(Long.parseLong(splited[0].trim())));  
            String rowKeyString = splited[1]+":"+formatedDate;  
            out.set(rowKeyString+"\t"+line);  
            //13726230503:201706291728  1363157985066   13726230503 00-FD-07-A4-72-B8:CMCC  120.196.100.82  i02.c.aliimg.com        24  27  2481    24681   200
            context.write(key,out);  
        }  
    }  
    public static class LoadDataReducer extends TableReducer<LongWritable,Text,NullWritable>{  
        public static final String COLUMN_FAMILY = "cf";  
        @Override  
        protected void reduce(LongWritable key, Iterable<Text> values, Reducer<LongWritable, Text, NullWritable, Mutation>.Context context) throws IOException, InterruptedException {  

            for (Text tx : values) {  
                String[] splited = tx.toString().split("\t");  
                String rowkey = splited[0];  


                Put put = new Put(rowkey.getBytes());  
//                put.add(COLUMN_FAMILY.getBytes(), "raw".getBytes(), tx  
//                        .toString().getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "reportTime".getBytes(),  
                        splited[1].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "msisdn".getBytes(),  
                        splited[2].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "apmac".getBytes(),  
                        splited[3].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "acmac".getBytes(),  
                        splited[4].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "host".getBytes(),  
                        splited[5].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "siteType".getBytes(),  
                        splited[6].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "upPackNum".getBytes(),  
                        splited[7].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "downPackNum".getBytes(),  
                        splited[8].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "upPayLoad".getBytes(),  
                        splited[9].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "downPayLoad".getBytes(),  
                        splited[10].getBytes());  
                put.add(COLUMN_FAMILY.getBytes(), "httpStatus".getBytes(),  
                        splited[11].getBytes());  
                context.write(NullWritable.get(), put);  
            }  
        }  
    }  
    public static void createHBaseTable(String tableName) throws IOException {  

        HTableDescriptor htd = new HTableDescriptor(  
                TableName.valueOf(tableName));  
        HColumnDescriptor col = new HColumnDescriptor("cf");  
        htd.addFamily(col);  
        Configuration conf = HBaseConfiguration.create();  
        conf.set("hbase.zookeeper.quorum", "hive01");  
        HBaseAdmin admin = new HBaseAdmin(conf);  
        if (admin.tableExists(tableName)) {  
            System.out.println("table exists, trying to recreate table......");  
            admin.disableTable(tableName);  
            admin.deleteTable(tableName);  
        }  
        System.out.println("create new table:" + tableName);  
        admin.createTable(htd);  

    }  
    public static void main(String[] args) throws Exception {  
       args = new String[] { "hdfs://hive01:8020/input/hbase" };    
        Configuration conf = HBaseConfiguration.create();  
        // conf.set("hbaser.rootdir","hdfs://bigdata:8020/hbase");  
        conf.set("hbase.zookeeper.quorum", "hive01"); 

        conf.set(TableOutputFormat.OUTPUT_TABLE, "phone_log"); 

        createHBaseTable("phone_log"); 

        Job job = Job.getInstance(conf, "LoadData");  
        job.setJarByClass(LoadData.class);  
        job.setNumReduceTasks(1);  


        // 3.2 map class  
        job.setMapperClass(LoadDataMapper.class);  
        job.setMapOutputKeyClass(LongWritable.class);  
        job.setMapOutputValueClass(Text.class);  


        // 3.3 reduce class  
        job.setReducerClass(LoadDataReducer.class);  
      //  job.setOutputKeyClass(NullWritable.class);     --不需要设置  
     //   job.setOutputValueClass(Mutation.class);     --不需要设置  



        Path inPath = new Path(args[0]);  
        FileInputFormat.addInputPath(job, inPath);  

        job.setOutputFormatClass(TableOutputFormat.class);  


        System.exit(job.waitForCompletion(true) ? 0 : 1);  

    }  
}