storm directgrouping

转载于 2018-11-15 00:54:49 发布 · 237 阅读

4 篇文章

订阅专栏

本文深入探讨了Apache Storm中直接分组（Direct Grouping）的概念，解析了如何通过指定下游任务ID实现精确的数据流控制。提供了详细的代码示例，包括Spout、Bolt和Topology的实现，展示了直接分组在数据流处理中的应用。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

							<div class="article-copyright">
				版权声明：本文为博主原创文章，未经博主允许不得转载。					https://blog.youkuaiyun.com/Simon_09010817/article/details/81364357				</div>
							            <link rel="stylesheet" href="https://csdnimg.cn/release/phoenix/template/css/ck_htmledit_views-f76675cdea.css">
					<div class="htmledit_views">
            <p style="margin-left:0pt;">一、概述</p>

这种方式发送者可以指定下游的哪个任务可以接收这个元组。只有在数据流被声明为直接数据流时才能够使用直接分组方式。使用直接数据流发送元组需要使用 OutputCollector 的其中一个 emitDirect 方法。Bolt 可以通过 TopologyContext 来获取它的下游消费者的任务 id，也可以通过跟踪 OutputCollector 的 emit 方法（该方法会返回它所发送元组的目标任务的 id）的数据来获取任务 id。

二、代码

1.Spout


package com.test.csdn.directgrouping;
 
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
 
import java.util.List;
import java.util.Map;
import java.util.Random;
 
/**
 * Created by Simon on 2018/8/1.
 */
public class DirectGroupingSpout extends BaseRichSpout {
 
    private SpoutOutputCollector collector;
 
    private String[] str =  {"xiaomi","huawei","apple","oppo","vivo","lenovo","LG",
            "samsung","htc","honor","nokia","smartisan","Sony","BlackBerry","sharp"};
 
    private int count;
 
    List<Integer> componentTasks;
 
    @Override
    public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        this.collector=spoutOutputCollector;
        componentTasks = topologyContext.getComponentTasks("bolt");
    }
 
    @Override
    public void nextTuple() {
        while (count <= 10){
            try {
                Thread.sleep(1000);
                int i = new Random().nextInt(10);
                String string = str[i];
                collector.emitDirect(componentTasks.get(1),new Values(string));
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            count++;
        }
 
 
    }
 
    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("string"));
    }
}

2.Bolt


package com.test.csdn.directgrouping;
 
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;
 
/**
 * Created by Simon on 2018/8/1.
 */
public class DirectGroupingBolt extends BaseBasicBolt {
 
    @Override
    public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        System.out.println(Thread.currentThread().getName()+"___"+tuple.getValue(0));
    }
 
    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
 
    }
}

3.Topo


package com.test.csdn.directgrouping;
 
import com.test.csdn.globalgrouping.GlobalGroupingBolt;
import com.test.csdn.globalgrouping.GlobalGroupingSpout;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;
 
/**
 * Created by Simon on 2018/8/1.
 */
public class DirectGroupingTopo {
    public static void main(String[] args) {
 
        TopologyBuilder builder = new TopologyBuilder();
 
        builder.setSpout("spout", new DirectGroupingSpout(),2).setNumTasks(3);
        //指定3个，便于测试
        builder.setBolt("bolt", new DirectGroupingBolt(),3).setNumTasks(5).directGrouping("spout");
 
        Config conf = new Config();
        conf.setDebug(false);
 
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("DirectGroupingTopo", conf, builder.createTopology());
        Utils.sleep(Long.MAX_VALUE);
        cluster.shutdown();
 
    }
}

三、运行输出