云原生小课堂｜Envoy请求流程源码解析（三）：请求解析

​

前言

Envoy 是一款面向 Service Mesh 的高性能网络代理服务。它与应用程序并行运行，通过以平台无关的方式提供通用功能来抽象网络。当基础架构中的所有服务流量都通过 Envoy 网格时，通过一致的可观测性，很容易地查看问题区域，调整整体性能。

Envoy也是istio的核心组件之一，以 sidecar 的方式与服务运行在一起，对服务的流量进行拦截转发，具有路由，流量控制等等强大特性。本系列文章，我们将不局限于istio，envoy的官方文档，从源码级别切入，分享Envoy启动、流量劫持、http 请求处理流程的进阶应用实例，深度分析Envoy架构。

本篇将是Envoy请求流程源码解析的第三篇，主要分享Envoy的outbound方向下篇，包含：接收请求、发送请求、接收响应、返回响应。注：本文中所讨论的issue和pr基于21年12月。

outbound方向

接收请求

1. client开始向socket写入请求数据

2. eventloop在触发read event后，transport_socket_.doRead中会循环读取加入read_buffer_，直到返回EAGAIN

   void ConnectionImpl::onReadReady() {
     ENVOY_CONN_LOG(trace, "read ready. dispatch_buffered_data={}", *this, dispatch_buffered_data_);
     const bool latched_dispatch_buffered_data = dispatch_buffered_data_;
     dispatch_buffered_data_ = false;
    
     ASSERT(!connecting_);
    
     // We get here while read disabled in two ways.
     // 1) There was a call to setTransportSocketIsReadable(), for example if a raw buffer socket ceded
     //    due to shouldDrainReadBuffer(). In this case we defer the event until the socket is read
     //    enabled.
     // 2) The consumer of connection data called readDisable(true), and instead of reading from the
     //    socket we simply need to dispatch already read data.
     if (read_disable_count_ != 0) {
       // Do not clear transport_wants_read_ when returning early; the early return skips the transport
       // socket doRead call.
       if (latched_dispatch_buffered_data && filterChainWantsData()) {
         onRead(read_buffer_->length());
       }
       return;
     }
    
     // Clear transport_wants_read_ just before the call to doRead. This is the only way to ensure that
     // the transport socket read resumption happens as requested; onReadReady() returns early without
     // reading from the transport if the read buffer is above high watermark at the start of the
     // method.
     transport_wants_read_ = false;
     IoResult result = transport_socket_->doRead(*read_buffer_);
     uint64_t new_buffer_size = read_buffer_->length();
     updateReadBufferStats(result.bytes_processed_, new_buffer_size);
    
     // If this connection doesn't have half-close semantics, translate end_stream into
     // a connection close.
     if ((!enable_half_close_ && result.end_stream_read_)) {
       result.end_stream_read_ = false;
       result.action_ = PostIoAction::Close;
     }
    
     read_end_stream_ |= result.end_stream_read_;
     if (result.bytes_processed_ != 0 || result.end_stream_read_ ||
         (latched_dispatch_buffered_data && read_buffer_->length() > 0)) {
       // Skip onRead if no bytes were processed unless we explicitly want to force onRead for
       // buffered data. For instance, skip onRead if the connection was closed without producing
       // more data.
       onRead(new_buffer_size);
     }
    
     // The read callback may have already closed the connection.
     if (result.action_ == PostIoAction::Close || bothSidesHalfClosed()) {
       ENVOY_CONN_LOG(debug, "remote close", *this);
       closeSocket(ConnectionEvent::RemoteClose);
     }
   }

1. 把buffer传入Envoy::Http::ConnectionManagerImpl::onData进行HTTP请求的处理

    Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) {
      if (!codec_) {
        // Http3 codec should have been instantiated by now.
        createCodec(data);
      }
     
      bool redispatch;
      do {
        redispatch = false;
     
        const Status status = codec_->dispatch(data);
     
        if (isBufferFloodError(status) || isInboundFramesWithEmptyPayloadError(status)) {
          handleCodecError(status.message());
          return Network::FilterStatus::StopIteration;
        } else if (isCodecProtocolError(status)) {
          stats_.named_.downstream_cx_protocol_error_.inc();
          handleCodecError(status.message());
          return Network::FilterStatus::StopIteration;
        }
        ASSERT(status.ok());

1. 如果codec_type是AUTO（HTTP1,2，3目前还不支持，在计划中）的情况下，会判断请求是否以PRI * HTTP/2为开始来判断是否http2

    Http::ServerConnectionPtr
    HttpConnectionManagerConfig::createCodec(Network::Connection& connection,
                                             const Buffer::Instance& data,
                                             Http::ServerConnectionCallbacks& callbacks) {
      switch (codec_type_) {
      case CodecType::HTTP1: {
        return std::make_unique<Http::Http1::ServerConnectionImpl>(
            connection, Http::Http1::CodecStats::atomicGet(http1_codec_stats_, context_.scope()),
            callbacks, http1_settings_, maxRequestHeadersKb(), maxRequestHeadersCount(),
            headersWithUnderscoresAction());
      }
      case CodecType::HTTP2: {
        return std::make_unique<Http::Http2::ServerConnectionImpl>(
            connection, callbacks,
            Http::Http2::CodecStats::atomicGet(http2_codec_stats_, context_.scope()),
            context_.api().randomGenerator(), http2_options_, maxRequestHeadersKb(),
            maxRequestHeadersCount(), headersWithUnderscoresAction());
      }
      case CodecType::HTTP3:
    #ifdef ENVOY_ENABLE_QUIC
        return std::make_unique<Quic::QuicHttpServerConnectionImpl>(
            dynamic_cast<Quic::EnvoyQuicServerSession&>(connection), callbacks,
            Http::Http3::CodecStats::atomicGet(http3_codec_stats_, context_.scope()), http3_options_,
            maxRequestHeadersKb(), headersWithUnderscoresAction());
    #else
        // Should be blocked by configuration checking at an earlier point.
        NOT_REACHED_GCOVR_EXCL_LINE;
    #endif
      case CodecType::AUTO:
        return Http::ConnectionManagerUtility::autoCreateCodec(
            connection, data, callbacks, context_.scope(), context_.api().randomGenerator(),
            http1_codec_stats_, http2_codec_stats_, http1_settings_, http2_options_,
            maxRequestHeadersKb(), maxRequestHeadersCount(), headersWithUnderscoresAction());
      }
      NOT_REACHED_GCOVR_EXCL_LINE;
    }
     
     
    std::string ConnectionManagerUtility::determineNextProtocol(Network::Connection& connection,
                                                                const Buffer::Instance& data) {
      if (!connection.nextProtocol().empty()) {
        return connection.nextProtocol();
      }
     
      // See if the data we have so far shows the HTTP/2 prefix. We ignore the case where someone sends
      // us the first few bytes of the HTTP/2 prefix since in all public cases we use SSL/ALPN. For
      // internal cases this should practically never happen.
      if (data.startsWith(Http2::CLIENT_MAGIC_PREFIX)) {
        return Utility::AlpnNames::get().Http2;
      }
     
      return "";
    }
     
     
    const std::string CLIENT_MAGIC_PREFIX = "PRI * HTTP/2";

1. 利用http_parser进行http解析的callback,ConnectionImpl::settings_静态初始化了parse各个阶段的callbacks

    http_parser_settings ConnectionImpl::settings_{
        [](http_parser* parser) -> int {
          static_cast<ConnectionImpl*>(parser->data)->onMessageBeginBase();
          return 0;
        },
        [](http_parser* parser, const char* at, size_t length) -> int {
          static_cast<ConnectionImpl*>(parser->data)->onUrl(at, length);
          return 0;
        },
        nullptr, // on_status
        [](http_parser* parser, const char* at, size_t length) -> int {
          static_cast<ConnectionImpl*>(parser->data)->onHeaderField(at, length);
          return 0;
        },
        [](http_parser* parser, const char* at, size_t length) -> int {
          static_cast<ConnectionImpl*>(parser->data)->onHeaderValue(at, length);
          return 0;
        },
        [](http_parser* parser) -> int {
          return static_cast<ConnectionImpl*>(parser->data)->onHeadersCompleteBase();
        },
        [](http_parser* parser, const char* at, size_t length) -> int {
          static_cast<ConnectionImpl*>(parser->data)->onBody(at, length);
          return 0;
        },
        [](http_parser* parser) -> int {
          static_cast<ConnectionImpl*>(parser->data)->onMessageCompleteBase();
          return 0;
        },
        nullptr, // on_chunk_header
        nullptr  // on_chunk_complete
    };

envoy社区有讨论会将协议解析器从http_parser换成llhttp

• https://github.com/envoyproxy/envoy/issues/5155

• https://github.com/envoyproxy/envoy/pull/15263/files 使用解析器接口，重构http parser

• https://github.com/envoyproxy/envoy/pull/15814添加llhttp解析器的实现，暂时还没合并

    if (pos != absl::string_view::npos) {
          // Include \r or \n
          new_data = new_data.substr(0, pos + 1);
          ssize_t rc = http_parser_execute(&parser_, &settings_, new_data.data(), new_data.length());
          ENVOY_LOG(trace, "http inspector: http_parser parsed {} chars, error code: {}", rc,
                    HTTP_PARSER_ERRNO(&parser_));
     
          // Errors in parsing HTTP.
          if (HTTP_PARSER_ERRNO(&parser_) != HPE_OK && HTTP_PARSER_ERRNO(&parser_) != HPE_PAUSED) {
            return ParseState::Error;
          }
     
          if (parser_.http_major == 1 && parser_.http_minor == 1) {
            protocol_ = Http::Headers::get().ProtocolStrings.Http11String;
          } else {
            // Set other HTTP protocols to HTTP/1.0
            protocol_ = Http::Headers::get().ProtocolStrings.Http10String;
          }
          return ParseState::Done;
        } else {
          ssize_t rc = http_parser_execute(&parser_, &settings_, new_data.data(), new_data.length());
          ENVOY_LOG(trace, "http inspector: http_parser parsed {} chars, error code: {}", rc,
                    HTTP_PARSER_ERRNO(&parser_));
     
          // Errors in parsing HTTP.
          if (HTTP_PARSER_ERRNO(&parser_) != HPE_OK && HTTP_PARSER_ERRNO(&parser_) != HPE_PAUSED) {
            return ParseState::Error;
          } else {
            return ParseState::Continue;
          }
     
     
     
     
        return {http_parser_execute(&parser_, &settings_, slice, len), HTTP_PARSER_ERRNO(&parser_)};

1. onMessageBeginBase

     current_header_map_ = std::make_unique<HeaderMapImpl>();
      header_parsing_state_ = HeaderParsingState::Field;
     
     
     
    Status ConnectionImpl::onMessageBegin() {
      ENVOY_CONN_LOG(trace, "message begin", connection_);
      // Make sure that if HTTP/1.0 and HTTP/1.1 requests share a connection Envoy correctly sets
      // protocol for each request. Envoy defaults to 1.1 but sets the protocol to 1.0 where applicable
      // in onHeadersCompleteBase
      protocol_ = Protocol::Http11;
      processing_trailers_ = false;
      header_parsing_state_ = HeaderParsingState::Field;
      allocHeaders(statefulFormatterFromSettings(codec_settings_));
      return onMessageBeginBase();
    }
     
    Status ServerConnectionImpl::onMessageBeginBase() {
      if (!resetStreamCalled()) {
        ASSERT(!active_request_.has_value());
        active_request_.emplace(*this);
        auto& active_request = active_request_.value();
        if (resetStreamCalled()) {
          return codecClientError("cannot create new streams after calling reset");
        }
        active_request.request_decoder_ = &callbacks_.newStream(active_request.response_encoder_);
     
        // Check for pipelined request flood as we prepare to accept a new request.
        // Parse errors that happen prior to onMessageBegin result in stream termination, it is not
        // possible to overflow output buffers with early parse errors.
        RETURN_IF_ERROR(doFloodProtectionChecks());
      }
      return okStatus();
    }
    

• 创建ActiveStream, 保存downstream的信息，和对应的route信息

• 对于https，会把TLS握手的时候保存的SNI写入ActiveStream.requested_server_name_

       void setRequestedServerName(absl::string_view requested_server_name) override {
           requested_server_name_ = std::string(requested_server_name);
         }
        
       void Filter::onServername(absl::string_view name) {
         if (!name.empty())&nb