package com.word;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
public class WordCountDemo {
public static void main(String[] args) {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS","hdfs://node01:8020");
configuration.set("yarn.resourcemanager.hostname", "node03:8808");
configuration.set("hbase.zookeeper.quorum", "node01,node02,node03");
try {
Job job = Job.getInstance(configuration);
job.setJarByClass(WordCountDemo.class);
job.setMapperClass(WCMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path("/WC/input/word.txt"));
TableMapReduceUtil.initTableReducerJob("wc", WCReducer.class, job);
if(job.waitForCompletion(true)){
System.out.println("~~~~~~~ ok ~~~~~~~");
}
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
package com.word;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
public class WCMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String lines = value.toString();
StringTokenizer words = new StringTokenizer(lines);
while(words.hasMoreTokens()){
context.write(new Text(words.nextToken()), new IntWritable(1));
}
}
}
package com.sxt.wc;
import java.io.IOException;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
public class WCReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> {
@Override
protected void reduce(Text text, Iterable<IntWritable> iterable,
Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable i : iterable) {
sum += i.get();
}
System.out.println("============="+text.toString());
Put put = new Put(text.toString().getBytes());
put.add("cf1".getBytes(), "count".getBytes(), (sum+"").getBytes());
context.write(null, put);
}
}