Flume LoadBalanceSinkProcessor源码阅读记录

简介

LoadBalanceSinkProcessor是flume实现负载均衡的重要方式，可有效的提高资源利用率，提升数据传输的速度。

配置参数

参数	默认值	描述
type	-	load_balance
selector	round_robin	round_robin或random
backoff	false	若为true,失败的Sink将加入黑名单，时间以指数增长
selector.maxTimeOut	30000(ms)	黑名单最大时间

配置示例

agent.sinks = s1 s2 s3 s4
agent.sinkgroups.group1.processor.type = failover
agent.sinkgroups.group1.processor.priority.s1 = 90
agent.sinkgroups.group1.processor.priority.s2 = 100
agent.sinkgroups.group1.processor.priority.s4 = 110
agent.sinkgroups.group1.processor.maxpenalty = 10000

LoadBalanceSinkProcessor初始化

  @Override
  public void configure(Context context) {
    Preconditions.checkState(getSinks().size() > 1,
        "The LoadBalancingSinkProcessor cannot be used for a single sink. "
        + "Please configure more than one sinks and try again.");
    //选初始化择器类型，默认为ROUND_ROBIN
    String selectorTypeName = context.getString(CONFIG_SELECTOR,
        SELECTOR_NAME_ROUND_ROBIN);
	//是否配置失败加入黑名单
    Boolean shouldBackOff = context.getBoolean(CONFIG_BACKOFF, false);

    selector = null;

    if (selectorTypeName.equalsIgnoreCase(SELECTOR_NAME_ROUND_ROBIN)) {
      selector = new RoundRobinSinkSelector(shouldBackOff);
    } else if (selectorTypeName.equalsIgnoreCase(SELECTOR_NAME_RANDOM)) {
      selector = new RandomOrderSinkSelector(shouldBackOff);
    } else {
      try {
        @SuppressWarnings("unchecked")
        Class<? extends SinkSelector> klass = (Class<? extends SinkSelector>)
            Class.forName(selectorTypeName);

        selector = klass.newInstance();
      } catch (Exception ex) {
        throw new FlumeException("Unable to instantiate sink selector: "
            + selectorTypeName, ex);
      }
    }
    //把配置的Sink列表加入到选择器中
    selector.setSinks(getSinks());
    selector.configure(
        new Context(context.getSubProperties(CONFIG_SELECTOR_PREFIX)));

    LOGGER.debug("Sink selector: " + selector + " initialized");
  }

选择器类型

1. RandomOrderSelector选择器

RandomOrderSelector，通过随机选择的方式，提供Sink名单。实现方法如下:

  @Override
  public synchronized Iterator<T> createIterator() {
    List<Integer> indexList = getIndexList();

    int size = indexList.size();
    int[] indexOrder = new int[size];

    while (indexList.size() != 1) {
      int pick = random.nextInt(indexList.size());
      indexOrder[indexList.size() - 1] = indexList.remove(pick);
    }

    indexOrder[0] = indexList.get(0);

    return new SpecificOrderIterator<T>(indexOrder, getObjects());
  }

2. RoundRobinOrderSelector选择器

RandomOrderSelector，通过顺序循环的方式，提供Sink名单。实现方法如下:

 @Override
  public Iterator<T> createIterator() {
    List<Integer> activeIndices = getIndexList();
    int size = activeIndices.size();
    // possible that the size has shrunk so gotta adjust nextHead for that
    if (nextHead >= size) {
      nextHead = 0;
    }
    int begin = nextHead++;
    if (nextHead == activeIndices.size()) {
      nextHead = 0;
    }

    int[] indexOrder = new int[size];

    for (int i = 0; i < size; i++) {
      indexOrder[i] = activeIndices.get((begin + i) % size);
    }

    return new SpecificOrderIterator<T>(indexOrder, getObjects());
  }

LoadBalanceSinkProcessor处理数据

 @Override
 public Status process() throws EventDeliveryException {
   Status status = null;
   //从选择器中获取Sink迭代器，此时的Sink处理顺序以由选择器做好
   Iterator<Sink> sinkIterator = selector.createSinkIterator();
   while (sinkIterator.hasNext()) {
     Sink sink = sinkIterator.next();
     try {
       //处理数据
       status = sink.process();
       break;
     } catch (Exception ex) {
       //处理失败，则提示处理器将其加入黑名单
       selector.informSinkFailed(sink);
       LOGGER.warn("Sink failed to consume event. "
           + "Attempting next sink if available.", ex);
     }
   }

   if (status == null) {
     throw new EventDeliveryException("All configured sinks have failed");
   }

   return status;
 }