CheckpointedFunction

CheckpointedFunction&CheckpointListener

作为一个接口，CheckpointedFunction提供如下两个方法：

//获得function或者operator的当前状态（快照）
void snapshotState(FunctionSnapshotContext context) throws Exception;
//用于从之前的检查点中恢复function或operator的状态
void initializeState(FunctionInitializationContext context) throws Exception;

继承关系如下图：

CheckpointListener是一个监听器，它会在某个检查点完成之后给出通知，客户程序可以订阅它然后进行相应的回调处理notifyCheckpointComplete

FlinkKafkaConsumerBase

FlinkKafkaConsumerBase是Kafka Source的基础类，内部实现了CheckpointedFunction接口，这属于Operator State的一种，下面我们看看具体方法是如何实现CheckPoint的

//FlinkKafkaConsumerBase
	public final void initializeState(FunctionInitializationContext context) throws Exception {
		OperatorStateStore stateStore = context.getOperatorStateStore();
		//从stateStore中恢复上一次提交的位置
		ListState<Tuple2<KafkaTopicPartition, Long>> oldRoundRobinListState =
		stateStore.getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);
		this.unionOffsetStates = stateStore.getUnionListState(new ListStateDescriptor<>(
				OFFSETS_STATE_NAME,
				TypeInformation.of(new TypeHint<Tuple2<KafkaTopicPartition, Long>>() {})));
		if (context.isRestored() && !restoredFromOldState) {
			restoredState = new TreeMap<>(new KafkaTopicPartition.Comparator());
			for (Tuple2<KafkaTopicPartition, Long> kafkaOffset : oldRoundRobinListState.get()) {
				restoredFromOldState = true;
				unionOffsetStates.add(kafkaOffset);
			}
			oldRoundRobinListState.clear();
			}
			for (Tuple2<KafkaTopicPartition, Long> kafkaOffset : unionOffsetStates.get()) {
				restoredState.put(kafkaOffset.f0, kafkaOffset.f1);
			}
		} 
		...
	}

    public final void snapshotState(FunctionSnapshotContext context) throws Exception {
		if (!running) {
			...
		} else {
		//清空unionOffsetStates
			unionOffsetStates.clear();
			final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
			//fetcher为空
			if (fetcher == null) {
				for (Map.Entry<KafkaTopicPartition, Long> subscribedPartition : subscribedPartitionsToStartOffsets.entrySet()) {
				//遍历subscribedPartitionsToStartOffsets，将key和value加入unionOffsetStates
					unionOffsetStates.add(Tuple2.of(subscribedPartition.getKey(), subscribedPartition.getValue()));
				}
				//如果提交模式为ON_CHECKPOINTS，只有当快照已经完成才会提交offset
				//将restoredState加入pendingOffsetsToCommit中
				if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
					pendingOffsetsToCommit.put(context.getCheckpointId(), restoredState);
				}
			} else {
			//fetcher不为空，从snapshotCurrentState中获取当前的offset
				HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState();
				if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
					pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);
				}
				for (Map.Entry<KafkaTopicPartition, Long> kafkaTopicPartitionLongEntry : currentOffsets.entrySet()) {
					unionOffsetStates.add(
							Tuple2.of(kafkaTopicPartitionLongEntry.getKey(), kafkaTopicPartitionLongEntry.getValue()));
				}
			}
			if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
			//当pendingOffsetsToCommit大小超过MAX_NUM_PENDING_CHECKPOINTS默认100时，删除第1个元素，防止发生OOM
				while (pendingOffsetsToCommit.size() > MAX_NUM_PENDING_CHECKPOINTS) {
					pendingOffsetsToCommit.remove(0);
				}
			}
		}
	}

	public final void notifyCheckpointComplete(long checkpointId) throws Exception {
		if (!running) {
			//返回
		}
		final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
		if (fetcher == null) {
			//返回
		}
		if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
		...
			try {
				//查找checkpointId对应的index
				final int posInMap = pendingOffsetsToCommit.indexOf(checkpointId);
				if (posInMap == -1) {
					//返回
				}
				@SuppressWarnings("unchecked")
				//移除该索引对应的快照信息
				Map<KafkaTopicPartition, Long> offsets =
					(Map<KafkaTopicPartition, Long>) pendingOffsetsToCommit.remove(posInMap);
				//所有的检查点都是按时间递增有序的
				for (int i = 0; i < posInMap; i++) {
					pendingOffsetsToCommit.remove(0);
				}
				if (offsets == null || offsets.size() == 0) {
					//返回
				}
				//提交offset到kafka
				fetcher.commitInternalOffsetsToKafka(offsets, offsetCommitCallback);
			} catch (Exception e) {
				if (running) {
					throw e;
				}
			}
		}
	}

AbstractStreamOperator

AbstractStreamOperator是所有stream operator的基类，实现了StreamOperator接口，StreamOperator只实现了CheckpointListener，并没有实现CheckpointedFunction，为什么呢？
在StreamOperator中也定义了snapshotState和initializeState方法。看下AbstractStreamOperator中这两个方法以及notifyCheckpointComplete是如何实现的。

//AbstractStreamOperator
	public final OperatorSnapshotFutures snapshotState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions,
			CheckpointStreamFactory factory) throws Exception {
			...
		try (StateSnapshotContextSynchronousImpl snapshotContext = new StateSnapshotContextSynchronousImpl(
				checkpointId,
				timestamp,
				factory,
				keyGroupRange,
				getContainingTask().getCancelables())) {
			snapshotState(snapshotContext);
			//利用DefaultOperatorStateBackend来保存快照
			if (null != operatorStateBackend) {
				snapshotInProgress.setOperatorStateManagedFuture(
					operatorStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
			}
			//利用第三方的backend来保存快照，如内存或RocksDb
			if (null != internalStateBackend) {
				snapshotInProgress.setKeyedStateManagedFuture(
					internalStateBackend.snapshot(checkpointId, timestamp, factory, checkpointOptions));
			}
		} catch (Exception snapshotException) {
			...
		}
		return snapshotInProgress;
	}

	public void snapshotState(StateSnapshotContext context) throws Exception {
		if (getKeyContext() != null) {
			KeyedStateCheckpointOutputStream out;
			try {
				out = context.getRawKeyedOperatorStateOutput();
			} catch (Exception exception) {
			}
			try {
				KeyGroupsList allKeyGroups = out.getKeyGroupList();
				for (int keyGroupIdx : allKeyGroups) {
					out.startNewKeyGroup(keyGroupIdx);
					timeServiceManager.snapshotStateForKeyGroup(
						new DataOutputViewStreamWrapper(out), keyGroupIdx);
				}
			} catch (Exception exception) {
			} finally {
			}
		}
	}
	
	public void notifyCheckpointComplete(long checkpointId) throws Exception {
	//如果internalStateBackend 不为空，则调用internalStateBackend的notifyCheckpointComplete来通知检查点已完成
		if (internalStateBackend != null) {
			internalStateBackend.notifyCheckpointComplete(checkpointId);
		}
	}

operatorStateBackend为DefaultOperatorStateBackend，用来保存Operator的状态，只保存在内存中，internalStateBackend包含HeapInternalStateBackend和RocksDBInternalStateBackend，用来保存KeydState