Storm入门程序——WordCount

最新推荐文章于 2023-08-15 15:32:29 发布

原创最新推荐文章于 2023-08-15 15:32:29 发布 · 917 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#storm #wordcount

Storm 专栏收录该内容

3 篇文章

订阅专栏

本文介绍了一个使用Apache Storm实现的简单实时词频统计系统。该系统包括三个主要组件：WordReader Spout用于从指定路径读取文本文件并发送数据；WordSpliter Bolt将接收到的文本行分割成单词；WordCounter Bolt统计每个单词出现的次数，并定期输出统计结果。

spout：

WordReader：

package spout;

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;

/**
Author:Benjamin  
 */
public class WordReader extends BaseRichSpout{


    //SpoutOutputCollector 发射器，负责向外面发射数据
    //conf 配置信息
    //private static final long serialVersionUID = 2756293133996402960L;
    private String inputpath;
    private SpoutOutputCollector collector;
    //当一个task被初始化时会调用此open方法
    public void open(Map conf, TopologyContext context,
            SpoutOutputCollector collector) {
         inputpath = (String) conf.get("INPUTPATH");
         this.collector = collector;
    }
    //这个方法会被一直调用
    public void nextTuple() {

        Collection<File> listFiles = FileUtils.listFiles(new File(inputpath), new String[]{"txt"}, true);
        for (File file : listFiles) {
            try {
                List<String> readLines = FileUtils.readLines(file);
                for (String line : readLines) {
                    this.collector.emit(new Values(line));
                }
                FileUtils.moveFile(file, new File(file.getAbsolutePath()+System.currentTimeMillis()));
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    //声明发射的字段
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("line"));
    }

}

Bolt：

WordSpliter：

package bolt;

import java.util.Map;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

/**
Author:Benjamin  
 */
public class WordSpliter extends BaseRichBolt{


    //private static final long serialVersionUID = -3411512160818196264L;


    OutputCollector collector;
    @Override
    public void prepare(Map stormConf, TopologyContext context,
            OutputCollector collector) {
        this.collector = collector;
    }

    //一直执行
    @Override
    public void execute(Tuple input) {
        String stringByField = input.getStringByField("line");
        String[] split = stringByField.split(" ");
        for (String word : split) {
            this.collector.emit(new Values(word));
        }
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("word"));
    }   
}

WordCounter:

package bolt;

import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;

/**
Author:Benjamin  
 */
public class WordCounter extends BaseRichBolt{

    /**
     * 
     */
    //private static final long serialVersionUID = -8840777246416063904L;

    OutputCollector collector;
    HashMap<String, Integer> hashmap = new HashMap<String, Integer>();
    @Override
    public void prepare(Map stormConf, TopologyContext context,
            OutputCollector collector) {
        this.collector = collector;
        //每三秒统计一次
        new Thread(new Runnable() {

            @Override
            public void run() {

                while(true){
                    System.out.println("=======================");
                    for (Entry<String, Integer> entry : hashmap.entrySet()) {
                        System.out.println(entry.getKey()+"====="+entry.getValue());
                    }
                    try {
                        Thread.sleep(3000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }).start();

    }

    @Override
    public void execute(Tuple input) {
        String word = input.getStringByField("word");
        Integer integer = hashmap.get(word);

        if(integer == null){
            hashmap.put(word, 1);
        }else {
            hashmap.put(word, integer+1);
        }


    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {

    }
}

Topology:

package topolopy;

import spout.WordReader;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
import bolt.WordCounter;
import bolt.WordSpliter;

/**
Author:Benjamin  
 */
public class WordCountTopo {

    public static void main(String[] args) {

        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("wordreader", new WordReader());
        topologyBuilder.setBolt("wordspliter", new WordSpliter()).shuffleGrouping("wordreader");
        topologyBuilder.setBolt("wordcounter", new WordCounter()).shuffleGrouping("wordspliter");

        Config config = new Config();
        config.put("INPUTPATH", "j:\\inputdir");
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("wordcount", config, topologyBuilder.createTopology());
    }

}