Storm Bolt中读取Tuple数据

Storm Tuple 数据处理

最新推荐文章于 2020-06-30 17:29:05 发布

原创最新推荐文章于 2020-06-30 17:29:05 发布 · 570 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#大数据

Storm 专栏收录该内容

25 篇文章

订阅专栏

本文介绍Apache Storm中Tuple接口的功能及其实现方式，包括如何通过Tuple读取数据、根据字段名和下标获取数据等关键操作。

Tuple接口有很多方法可以读取从上游组件发送过来的数据，这些方法可以分为2类。

根据下标获取数据
根据字段名获取数据

读取数据方法

public class TupleImpl extends IndifferentAccessMap implements Seqable, Indexed, IMeta, Tuple {
    private List<Object> values;
    private int taskId;
    private String streamId;
    private GeneralTopologyContext context;
    private MessageId id;
    private IPersistentMap _meta = null;
    
    public TupleImpl(GeneralTopologyContext context, List<Object> values, int taskId, String streamId, MessageId id) {
        this.values = values;
        this.taskId = taskId;
        this.streamId = streamId;
        this.id = id;
        this.context = context;
        
        String componentId = context.getComponentId(taskId);
        Fields schema = context.getComponentOutputFields(componentId, streamId);
        if(values.size()!=schema.size()) {
            throw new IllegalArgumentException(
                    "Tuple created with wrong number of fields. " +
                    "Expected " + schema.size() + " fields but got " +
                    values.size() + " fields");
        }
    }

    public TupleImpl(GeneralTopologyContext context, List<Object> values, int taskId, String streamId) {
        this(context, values, taskId, streamId, MessageId.makeUnanchored());
    }    
    
    Long _processSampleStartTime = null;
    Long _executeSampleStartTime = null;
	
	//从任务的上下文【任务创建时定义好的】里获取Tuple定义的Fields
    public Fields getFields() {
        return context.getComponentOutputFields(getSourceComponent(), getSourceStreamId());
    }
	
	//获取到field的下标，直接在values【List】中获取对应的数据
	public Object getValueByField(String field) {
        return values.get(fieldIndex(field));
    }
	
	public String getStringByField(String field) {
        return (String) values.get(fieldIndex(field));
    }
	
	//根据下标直接在Values[List]中获取数据
	public Object getValue(int i) {
        return values.get(i);
    }
     
    //.......	 
	
}

任务上下文

public class GeneralTopologyContext implements JSONAware {
    private StormTopology _topology;
    private Map<Integer, String> _taskToComponent;
    private Map<String, List<Integer>> _componentToTasks;
    private Map<String, Map<String, Fields>> _componentToStreamToFields;
    private String _stormId;
    protected Map _stormConf;
    
    // pass in componentToSortedTasks for the case of running tons of tasks in single executor
    public GeneralTopologyContext(StormTopology topology, Map stormConf,
            Map<Integer, String> taskToComponent, Map<String, List<Integer>> componentToSortedTasks,
            Map<String, Map<String, Fields>> componentToStreamToFields, String stormId) {
        _topology = topology;
        _stormConf = stormConf;
        _taskToComponent = taskToComponent;
        _stormId = stormId;
        _componentToTasks = componentToSortedTasks;
        _componentToStreamToFields = componentToStreamToFields;
    }
	    /**
     * Gets the declared output fields for the specified component/stream.
     */
    public Fields getComponentOutputFields(String componentId, String streamId) {
        Fields ret = _componentToStreamToFields.get(componentId).get(streamId);
        if(ret==null) {
            throw new IllegalArgumentException("No output fields defined for component:stream " + componentId + ":" + streamId);
        }
        return ret;
    }
	
     //......	
	
}