etcd watch 实现原理

介绍

在 etcd 中，watch 是一个非常重要的特性，它可以让客户端监控 etcd 中的 key 或者一组 key，当 key 发生变化时，etcd 会通知客户端。本文将介绍 etcd watch 的实现原理。

etcdctl watch /test
# 当 /test 的值发生变化时，会输出如下信息
PUT
/test
a
PUT
/test
b
DELETE
/test

watch 的 api

etcd watch api 是由 grpc stream 实现的，客户端通过 grpc stream 发送 watch 请求，etcd 会将 key 的变化通过 stream 返回给客户端。

rpc Watch(stream WatchRequest) returns (stream WatchResponse) {
      option (google.api.http) = {
        post: "/v3/watch"
        body: "*"
    };
}

api 实现

func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) {
   
   
	sws := serverWatchStream{
   
   
		lg: ws.lg,

		clusterID: ws.clusterID,
		memberID:  ws.memberID,

		maxRequestBytes: ws.maxRequestBytes,

		sg:        ws.sg,
		watchable: ws.watchable,
		ag:        ws.ag,

		gRPCStream:  stream,
		watchStream: ws.watchable.NewWatchStream(),
		// chan for sending control response like watcher created and canceled.
		ctrlStream: make(chan *pb.WatchResponse, ctrlStreamBufLen),

		progress: make(map[mvcc.WatchID]bool),
		prevKV:   make(map[mvcc.WatchID]bool),
		fragment: make(map[mvcc.WatchID]bool),

		closec: make(chan struct{
   
   }),
	}

	sws.wg.Add(1)
	go func() {
   
   
        // 开启一个 goroutine 处理新的 event 然后发送给客户端
		sws.sendLoop()
		sws.wg.Done()
	}()

	errc := make(chan error, 1)
	
	go func() {
   
   
        // 开启一个 goroutine 处理客户端发送的 watch 请求
		if rerr := sws.recvLoop(); rerr != nil {
   
   
			if isClientCtxErr(stream.Context().Err(), rerr) {
   
   
				sws.lg.Debug("failed to receive watch request from gRPC stream", zap.Error(rerr))
			} else {
   
   
				sws.lg.Warn("failed to receive watch request from gRPC stream", zap.Error(rerr))
				streamFailures.WithLabelValues("receive", "watch").Inc()
			}
			errc <- rerr
		}
	}()

	// 处理结束
	select {
   
   
	case err = <-errc:
		if err == context.Canceled {
   
   
			err = rpctypes.ErrGRPCWatchCanceled
		}
		close(sws.ctrlStream)
	case <-stream.Context().Done():
		err = stream.Context().Err()
		if err == context.Canceled {
   
   
			err = rpctypes.ErrGRPCWatchCanceled
		}
	}

	sws.close()
	return err
}

这里 主要的逻辑是开启两个 goroutine，一个用于处理客户端发送的 watch 请求，另一个用于处理新的 event 然后发送给客户端。

sendLoop

func (sws *serverWatchStream) sendLoop() {
   
   
	// watch ids that are currently active
	ids := make(map[mvcc.WatchID]struct{
   
   })
	// watch responses pending on a watch id creation message
	pending := make(map[mvcc.WatchID][]*pb.WatchResponse)

	interval := GetProgressReportInterval()
	progressTicker := time.NewTicker(interval)

	defer func() {
   
   
		progressTicker.Stop()
		// 清空chan ，清理待处理 event
		for ws := range sws.watchStream.Chan() {
   
   
			mvcc.ReportEventReceived(len(ws.Events))
		}
		for _, wrs := range pending {
   
   
			for _, ws := range wrs {
   
   
				mvcc.ReportEventReceived(len(ws.Events))
			}
		}
	}()

	for {
   
   
		select {
   
   
		case wresp, ok := <-sws.watchStream.Chan():
            // 从 watchStream.Chan() 中获取 event
            // 然后发送给客户端 
			if !ok {
   
   
				return
			}

			evs := wresp.Events
			events := make([]*mvccpb.Event, len(evs))
			sws.mu.RLock()
			needPrevKV := sws.prevKV[wresp.WatchID]
			sws.mu.RUnlock()
			for i := range evs {
   
   
				events[i] = &evs[i]
				if needPrevKV && !IsCreateEvent(evs[i]) {
   
   
					opt := mvcc.RangeOptions{
   
   Rev: evs[i].Kv.ModRevision - 1}
					r, err := sws.watchable.Range(context.TODO(), evs[i].Kv.Key, nil, opt)
					if err == nil && len(r.KVs) != 0 {
   
   
						events[i].PrevKv = &(r.KVs[0])
					}
				}
			}

			canceled := wresp.CompactRevision != 0
			wr := &pb.WatchResponse{
   
   
				Header:          sws.newResponseHeader(wresp.Revision),
				WatchId:         int64(wresp.WatchID),
				Events:          events,
				CompactRevision: wresp.CompactRevision,
				Canceled:        canceled,
			}

			// Progress notifications can have WatchID -1
			// if they announce on behalf of multiple watchers
			if wresp.WatchID != clientv3.InvalidWatchID {
   
   
				if _, okID := ids[wresp.WatchID]; !okID {
   
   
					// buffer if id not yet announced
					wrs := append(pending[wresp.WatchID], wr)
					pending[wresp.WatchID] = wrs
					continue
				}
			}

			mvcc.ReportEventReceived(len(evs))

			sws.mu.RLock()
			fragmented, ok := sws.fragment[wresp.WatchID]
			sws.mu.RUnlock()

			var serr error
			// gofail: var beforeSendWatchResponse struct{}
			if !fragmented && !ok {
   
   
				serr = sws.gRPCStream.Send(wr)
			} else {
   
   
				serr = sendFragments(wr, sws.maxRequestBytes, sws.gRPCStream.Send)
			}

			if serr != nil {
   
   
				if isClientCtxErr(sws.gRPCStream.Context().Err(), serr) {
   
   
					sws.lg