背景
项目上基于apisix-runner,我们自己写了一些插件,执行点为ext-plugin-pre-req,按照apisix的官方文档上我们知道ext-plugin-req 的执行顺序是在大多数 APISIX built-in plugins之前;但是实际apisix的插件的执行点在阶段(phase)来说上划分的更为细化。
💡 ext-plugin-pre-req: executed during handling the request, before most of the APISIX built-in plugins (Lua language plugins)
问题
遇到的问题:在做链路追踪时,需要将zipkin插件的span和ext-*中的自定义插件的span关联
解决
按照ext-plugin-pre-req的定义,执行顺序是优先于大多数的apisix内置插件的,但是不包括zipkin,根据apisix的官方文档介绍,在相同的执行阶段,优先级高的先执行。
Plugin Develop | Apache APISIX® – Cloud-Native API Gateway
在conf/config-default.yaml配置中定义了plugins的优先级
plugins: # plugin list (sorted by priority)
- real-ip # priority: 23000
- client-control # priority: 22000
- proxy-control # priority: 21990
- zipkin # priority: 12011
#- skywalking # priority: 12010
#- opentelemetry # priority: 12009
- ext-plugin-pre-req # priority: 12000
- request-id # priority: 11010
- fault-injection # priority: 11000
- mocking # priority: 10900
- serverless-pre-function # priority: 10000
#- batch-requests # priority: 4010
- cors # priority: 4000
- ip-restriction # priority: 3000
- ua-restriction # priority: 2999
- referer-restriction # priority: 2990
- csrf # priority: 2980
- uri-blocker # priority: 2900
- request-validation # priority: 2800
- openid-connect # priority: 2599
...
apisix的lua脚本执行阶段根据源码,基本分为rewrite、access、header_filter,log
根据优先级来看zipkin 高于 ext-plugin-pre-req;
通过查看lua源码看到,在rewrite阶段,先执行了zipkin,后执行ext-plugin-pre-req;
-- apisix/apisix/plugins/zipkin.lua
-- span/v2中的“apisix.proxy”span 开启后,结束于header_filter阶段,而head中写入b3是发生在access阶段;
-- 这也就导致了我们ext-plugin-pre-req context中无法获取到span
-- 所以需要在rewrite阶段就写入b3
function _M.rewrite(plugin_conf, ctx)
....
local request_span = ctx.opentracing.request_span
if conf.span_version == ZIPKIN_SPAN_VER_1 then
ctx.opentracing.rewrite_span = request_span:start_child_span("apisix.rewrite",
start_timestamp)
ctx.REWRITE_END_TIME = tracer:time()
ctx.opentracing.rewrite_span:finish(ctx.REWRITE_END_TIME)
else
ctx.opentracing.proxy_span = request_span:start_child_span("apisix.proxy",
start_timestamp)
end
end
function _M.access(conf, ctx)
....
-- send headers to upstream
local outgoing_headers = {}
tracer:inject(opentracing.proxy_span, "http_headers", outgoing_headers)
for k, v in pairs(outgoing_headers) do
core.request.set_header(ctx, k, v)
end
end
function _M.header_filter(conf, ctx)
....
if conf.span_version == ZIPKIN_SPAN_VER_1 then
if opentracing.proxy_span then
opentracing.body_filter_span = opentracing.proxy_span:start_child_span(
"apisix.body_filter", end_time)
end
else
opentracing.proxy_span:finish(end_time)
opentracing.response_span = opentracing.request_span:start_child_span(
"apisix.response_span", end_time)
end
end
apisix/apisix/plugins/zipkin.lua 修改后代码
function _M.rewrite(plugin_conf, ctx)
....
local request_span = ctx.opentracing.request_span
if conf.span_version == ZIPKIN_SPAN_VER_1 then
ctx.opentracing.rewrite_span = request_span:start_child_span("apisix.rewrite",
start_timestamp)
ctx.REWRITE_END_TIME = tracer:time()
ctx.opentracing.rewrite_span:finish(ctx.REWRITE_END_TIME)
else
ctx.opentracing.proxy_span = request_span:start_child_span("apisix.proxy",
start_timestamp)
-- send headers to upstream
local outgoing_headers = {}
tracer:inject(ctx.opentracing.proxy_span, "http_headers", outgoing_headers)
for k, v in pairs(outgoing_headers) do
core.request.set_header(ctx, k, v)
end
end
end
在我们ext-plugin-pre-req 插件中关联“apisix-proxy” span
func (t *Token) Filter(conf interface{}, w http.ResponseWriter, r pkgHTTP.Request) {
var span opentracing.Span
carrier := opentracing.HTTPHeadersCarrier(r.Header().View())
clientContext, err := utils.Trace.Extract(opentracing.HTTPHeaders, carrier)
if err != nil {
log.Errorf("trace err:", err)
span = utils.Trace.StartSpan(t.Name())
} else {
span = utils.Trace.StartSpan(
t.Name(),
opentracing.ChildOf(clientContext))
}
defer span.Finish()
...
}