mapreduce将hdfs数据存入hbase-2.3.0

最新推荐文章于 2024-08-02 16:39:23 发布

原创最新推荐文章于 2024-08-02 16:39:23 发布 · 425 阅读

CC 4.0 BY-SA版权

本文详细介绍了如何从零开始搭建HBase集群，包括JDK与Zookeeper的安装配置，以及HBase的环境设置。此外，还提供了一个使用MapReduce进行WordCount并存储至HBase的代码示例。

<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-client</artifactId>
    <version>2.3.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-server</artifactId>
    <version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-mapreduce -->
<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-mapreduce</artifactId>
    <version>2.3.0</version>
</dependency>

1.首先搭建hbase集群

habse是基于zookper搭建的

先上传jdk的rpm文件

jdk-8u211-linux-x64.rpm peiyajie@10.202.234.56:/home/peiyajie
rpm -ivh jdk-8u211-linux-x64.rpm
sudo rpm -ivh jdk-8u211-linux-x64.rpm

然后完成安装、

解压zookper的安装文件

进入conf
cp zoo_sample.cfg zoo.cfg

在zoo.cfg增加：

server.2=10.202.234.233:2888:3888
server.3=10.202.234.56:2888:3888
server.1=10.202.234.244:2888:3888

修改

dataDir=/home/peiyajie/zookeeper-3.4.14/data

新建/home/peiyajie/zookeeper-3.4.14/data下面myid里面的值和集群的id对应

然后解压hhase的压缩文件

修改hbase-site.xml并且新增

<name>hbase.zookeeper.quorum</name>
<value>10.202.234.244,10.202.234.233,10.202.234.56</value>
<description>The directory shared by RegionServers.
</description>
</property>

<property>
<name>hbase.rootdir</name>
<value>hdfs://10.202.234.244:9000/hbase</value>
<description>The directory shared by RegionServers.
</description>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
<description>The mode the cluster will be in. Possible values are
false: standalone and pseudo-distributed setups with managed Zookeeper
true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
</description>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
</property>

在hbase-env.sh里面新增

export JAVA_HOME=/usr/java/jdk1.8.0_211-amd64
export HBASE_MANAGES_ZK=false

上代码：

package com.qihoo.hadoop.util;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WCHbaseMapper  extends  Mapper<LongWritable, Text,Text, NullWritable> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        context.write(value,NullWritable.get());
    }
}

package com.qihoo.hadoop.util;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;

import java.io.IOException;

/**
 *
 */
public class WCHbaseReducer extends TableReducer<Text, NullWritable, ImmutableBytesWritable> {
    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

        String[] split = key.toString().split(" ");
        Put put = new Put(Bytes.toBytes(split[0]));

        put.addColumn("info".getBytes(),"name".getBytes(),split[1].getBytes());
        put.addColumn("info".getBytes(),"age".getBytes(),Bytes.toBytes(Integer.parseInt(split[2])));

        context.write(new ImmutableBytesWritable(Bytes.toBytes(split[0])),put);
    }

}

package com.qihoo.hadoop.util;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import java.io.IOException;

public class HbaseUtil {
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException
    {
        Configuration conf = new Configuration();
        conf.set("hbase.zookeeper.quorum","10.202.234.244,10.202.234.233,10.202.234.56");
        conf.set("hbase.zookeeper.property.clientPort", "2181");
        Job job=Job.getInstance(conf, "Runner"); //

        Path path = new Path("hdfs://10.202.234.244:9000/wordcount/inputpeiyajie/3.txt");
        FileInputFormat.addInputPath(job, path);

        job.setJarByClass(HbaseUtil.class) ;

        job.setMapOutputKeyClass(ImmutableBytesWritable.class) ;
        job.setMapOutputValueClass(Put.class) ;

        job.setMapperClass(WCHbaseMapper.class) ;
      //  job.setReducerClass(WCHbaseReducer.class) ;

//    org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.setOutputPath(job, new Path("D:/testFile/mr1/" + new Date().getTime())) ;
        TableMapReduceUtil.initTableReducerJob("member", WCHbaseReducer.class, job) ;
        job.waitForCompletion(true);
    }

}