Apisix的ext-plugin-pre-req 中的trace 关联zipkin插件的trace

背景

项目上基于apisix-runner，我们自己写了一些插件，执行点为ext-plugin-pre-req，按照apisix的官方文档上我们知道ext-plugin-req 的执行顺序是在大多数 APISIX built-in plugins之前；但是实际apisix的插件的执行点在阶段（phase）来说上划分的更为细化。

💡 ext-plugin-pre-req: executed during handling the request, before most of the APISIX built-in plugins (Lua language plugins)

问题

遇到的问题：在做链路追踪时，需要将zipkin插件的span和ext-*中的自定义插件的span关联

解决

按照ext-plugin-pre-req的定义，执行顺序是优先于大多数的apisix内置插件的，但是不包括zipkin,根据apisix的官方文档介绍，在相同的执行阶段，优先级高的先执行。

Plugin Develop | Apache APISIX® – Cloud-Native API Gateway

在conf/config-default.yaml配置中定义了plugins的优先级

plugins:                          # plugin list (sorted by priority)
  - real-ip                        # priority: 23000
  - client-control                 # priority: 22000
  - proxy-control                  # priority: 21990
  - zipkin                         # priority: 12011
  #- skywalking                    # priority: 12010
  #- opentelemetry                 # priority: 12009
  - ext-plugin-pre-req             # priority: 12000
  - request-id                     # priority: 11010
  - fault-injection                # priority: 11000
  - mocking                        # priority: 10900
  - serverless-pre-function        # priority: 10000
  #- batch-requests                # priority: 4010
  - cors                           # priority: 4000
  - ip-restriction                 # priority: 3000
  - ua-restriction                 # priority: 2999
  - referer-restriction            # priority: 2990
  - csrf                           # priority: 2980
  - uri-blocker                    # priority: 2900
  - request-validation             # priority: 2800
  - openid-connect                 # priority: 2599
 ...

💡 注意这里的priority 在apisix的lua 源码中是写死的

apisix的lua脚本执行阶段根据源码，基本分为rewrite、access、header_filter，log

根据优先级来看zipkin 高于 ext-plugin-pre-req;

通过查看lua源码看到，在rewrite阶段，先执行了zipkin,后执行ext-plugin-pre-req;

-- apisix/apisix/plugins/zipkin.lua
-- span/v2中的“apisix.proxy”span 开启后，结束于header_filter阶段，而head中写入b3是发生在access阶段;
-- 这也就导致了我们ext-plugin-pre-req  context中无法获取到span
-- 所以需要在rewrite阶段就写入b3
function _M.rewrite(plugin_conf, ctx)
		....
		local request_span = ctx.opentracing.request_span
    if conf.span_version == ZIPKIN_SPAN_VER_1 then
        ctx.opentracing.rewrite_span = request_span:start_child_span("apisix.rewrite",
            start_timestamp)

        ctx.REWRITE_END_TIME = tracer:time()
        ctx.opentracing.rewrite_span:finish(ctx.REWRITE_END_TIME)
    else
        ctx.opentracing.proxy_span = request_span:start_child_span("apisix.proxy",
            start_timestamp)
    end
end

function _M.access(conf, ctx)
   ....

    -- send headers to upstream
    local outgoing_headers = {}
    tracer:inject(opentracing.proxy_span, "http_headers", outgoing_headers)
    for k, v in pairs(outgoing_headers) do
        core.request.set_header(ctx, k, v)
    end
end

function _M.header_filter(conf, ctx)
   ....
    if conf.span_version == ZIPKIN_SPAN_VER_1 then
        if opentracing.proxy_span then
            opentracing.body_filter_span = opentracing.proxy_span:start_child_span(
                "apisix.body_filter", end_time)
        end
    else
        opentracing.proxy_span:finish(end_time)
        opentracing.response_span = opentracing.request_span:start_child_span(
            "apisix.response_span", end_time)
    end
end

apisix/apisix/plugins/zipkin.lua 修改后代码

function _M.rewrite(plugin_conf, ctx)
		....
		local request_span = ctx.opentracing.request_span
		    if conf.span_version == ZIPKIN_SPAN_VER_1 then
		        ctx.opentracing.rewrite_span = request_span:start_child_span("apisix.rewrite",
		            start_timestamp)
		
		        ctx.REWRITE_END_TIME = tracer:time()
		        ctx.opentracing.rewrite_span:finish(ctx.REWRITE_END_TIME)
		    else
		        ctx.opentracing.proxy_span = request_span:start_child_span("apisix.proxy",
		            start_timestamp)
		
		        -- send headers to upstream
		        local outgoing_headers = {}
		        tracer:inject(ctx.opentracing.proxy_span, "http_headers", outgoing_headers)
		        for k, v in pairs(outgoing_headers) do
		            core.request.set_header(ctx, k, v)
		        end
		    end
		end

在我们ext-plugin-pre-req 插件中关联“apisix-proxy” span

func (t *Token) Filter(conf interface{}, w http.ResponseWriter, r pkgHTTP.Request) {
	var span opentracing.Span

	carrier := opentracing.HTTPHeadersCarrier(r.Header().View())
	clientContext, err := utils.Trace.Extract(opentracing.HTTPHeaders, carrier)
	if err != nil {
		log.Errorf("trace err:", err)
		span = utils.Trace.StartSpan(t.Name())
	} else {
		span = utils.Trace.StartSpan(
			t.Name(),
			opentracing.ChildOf(clientContext))
	}

	defer span.Finish()
	...
}

效果