storm to hbase demo

package com.mark.storm.project.hbasedemo;

import org.apache.hadoop.hbase.client.Durability;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria;
import org.apache.storm.hbase.trident.mapper.SimpleTridentHBaseMapper;
import org.apache.storm.hbase.trident.mapper.TridentHBaseMapper;
import org.apache.storm.hbase.trident.state.HBaseState;
import org.apache.storm.hbase.trident.state.HBaseStateFactory;
import org.apache.storm.hbase.trident.state.HBaseUpdater;
import org.apache.storm.topology.base.BaseWindowedBolt;
import org.apache.storm.trident.TridentTopology;
import org.apache.storm.trident.operation.BaseAggregator;
import org.apache.storm.trident.operation.BaseFunction;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.StateFactory;
import org.apache.storm.trident.testing.FixedBatchSpout;
import org.apache.storm.trident.tuple.TridentTuple;
import org.apache.storm.trident.windowing.config.SlidingDurationWindow;
import org.apache.storm.trident.windowing.config.WindowConfig;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;

import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * Created by fellowlei on 2018/3/6.
 */
public class StormToHbase {
    public static class Split extends BaseFunction {
        @Override
        public void execute(TridentTuple tuple, TridentCollector collector) {
            String sentence = tuple.getString(0);
            for (String word : sentence.split(" ")) {
                collector.emit(new Values(word));
            }
        }
    }

    public static class Aggregate extends BaseAggregator<Map<String,Integer>>{

        @Override
        public Map<String, Integer> init(Object o, TridentCollector tridentCollector) {
            return new HashMap<String,Integer>();
        }

        @Override
        public void aggregate(Map<String, Integer> map, TridentTuple tridentTuple, TridentCollector tridentCollector) {
            String word=tridentTuple.getStringByField("word");
            if(map.containsKey(word)){
                int num=map.get(word);
                map.put(word,num+1);
            }else {
                map.put(word,1);
            }
        }

        @Override
        public void complete(Map<String, Integer> map, TridentCollector tridentCollector) {
            List<Map.Entry<String,Integer>> list=new ArrayList<Map.Entry<String, Integer>>(map.entrySet());

            Collections.sort(list, new Comparator<Map.Entry<String,Integer>>() {
                @Override
                public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
                    return o2.getValue()-o1.getValue();
                }
            });

            for(int i=0;i<5&&i<list.size();i++){
                tridentCollector.emit(new Values(String.valueOf(i+1),list.get(i).getKey().toString(),list.get(i).getValue().toString()));
            }
        }
    }

    private static StormTopology buildTopology() {
        FixedBatchSpout fixedBatchSpout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
                new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
                new Values("how many apples can you eat"), new Values("to or not to be the person"));
        fixedBatchSpout.setCycle(true);


        //HBase相关配置
        //定义Mapper，把HBase的RowKey，列簇，列，对应到Trident中Tuple的Field
        //SimpleTridentHBaseMapper是TridentHBaseMapper的一个简单继承
        TridentHBaseMapper tridentHBaseMapper=new SimpleTridentHBaseMapper()
                .withColumnFamily("cf")
                .withColumnFields(new Fields("word","count"))
                .withRowKeyField("rank");

        //定义HBase数据到Tuple的投影，加入cf列族的word和count列
        HBaseProjectionCriteria projectionCriteria=new HBaseProjectionCriteria()
                .addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf","word"))
                .addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf","count"));

        //定义HbaseState的属性类Options
        HBaseState.Options options=new HBaseState.Options()
                .withConfigKey("hbase.conf")
                .withMapper(tridentHBaseMapper)
                .withDurability(Durability.SYNC_WAL)
//                .withProjectionCriteria(projectionCriteria)
                .withTableName("window");

        //使用工厂类生产HbaseState对象
        StateFactory factory=new HBaseStateFactory(options);

        //定义时间滑动窗口 每10秒计算过去30秒的数据
        WindowConfig slidingDurationWindow = SlidingDurationWindow.of(new BaseWindowedBolt.Duration(30, TimeUnit.SECONDS), new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS));


        //创建TridentTopology
        //HBaseUpdater是用于更新HBaseState的类
        //在滑动窗口中使用聚合操作
        TridentTopology topology = new TridentTopology();
        topology.newStream("spout", fixedBatchSpout)
                .each(new Fields("sentence"), new Split(), new Fields("word"))
                .window(slidingDurationWindow, new Fields("word"), new Aggregate(), new Fields("rank", "word", "count"))
                .partitionPersist(factory, new Fields("rank", "word", "count"), new HBaseUpdater(), new Fields());
        return topology.build();
    }

    public static void main(String[] args) {


        Config conf = new Config();
        conf.setDebug(true);
        conf.put("hbase.conf", new HashMap());
        LocalCluster cluster = new LocalCluster();

        cluster.submitTopology("topN", conf, buildTopology());

    }




}

http://blog.youkuaiyun.com/Nougats/article/details/72956864

http://blog.youkuaiyun.com/gpwner/article/details/72796337