nginx用lua脚本实现动态重载压测结果

压测结果（QPS）：

频率	无reload	5s	3s	1s	500ms	200ms	100ms
reload	35705.96	34784.15	33432.39	31955.77	28205.43	20778.90	16340.17
lua重载	29335.23	29185.19	29017.06	29579.67	29258.41	29230.13	29938.52
对比	-17.84%	-16.09%	-13.21%	-7.44%	3.73%	40.68%	83.22%

结论：

• reload方式随着频率的增加，QPS 明显下降。尤其在 500ms、200ms 和 100ms 重载频率下，性能影响逐渐显著，QPS 分别下降到 28205.43、20778.90 和 16340.17。最大降幅达到54%，如果频率继续提高，降幅会更显著。
• reload方式在 reload 期间，由于其会保留旧 worker 进程同时开启新的 worker 进程，导致 CPU 使用率显著下降，内存占用率显著上升（幅度均超过50%）。
• lua动态重载在不同频率下的表现稳定，QPS 维持在 29000-30000 左右，没有明显波动。
• 在频率不高（≥1秒）时，reload方式性能略优；但随着频率增加，动态重载表现出明显性能优势。

环境：

• 四台4C8G机器
• 客户端：1台（用于发起压测请求）
• NGINX：1台（使用 OpenResty，内置 Lua 模块）
• 后端服务：2台
• 压测工具：wrk；参数：4线程，4000连接

大致思路

在 worker 进程中，通过定时器定期检查并更新配置，将最新配置写入 NGINX 的共享内存，并利用 Lua 脚本实现轮询转发。

如果通过 HTTP 请求来实现配置更新，在高并发情况下可能会出现配置失效的情况。

性能调优

Lua 脚本还有代码优化空间，例如减少共享内存的set和flushAll操作、引入LuaJIT、减少日志等等，可以进一步降低 Lua 脚本的执行开销，提高整体 QPS 表现。

配置文件：

user  nginx;
worker_processes  4;

# 设置内存和文件描述符限制
worker_rlimit_core  1000M;       # 增大核心文件大小限制
worker_rlimit_nofile 65535;     # 增大文件描述符限制

pid        /var/run/nginx.pid;
error_log  error.log debug;

events {
    worker_connections  65535;
    multi_accept on;  # 每次尽可能接受多个新连接
}

http {
    lua_shared_dict endpoints_data 5m; #定义upstream共享内存空间
    lua_shared_dict cache 1m; #定义计数共享空间
    access_log  access.log;

    lua_package_path "/your/path/to/lua/?.lua;;";
    init_by_lua_block {
        collectgarbage("collect")
        local ok, res

        -- ngx.log(ngx.ERR, "Loading configuration")

        ok, res = pcall(require, "configuration")

        if not ok then

            ngx.log(ngx.ERR, "Failed to load configuration: " .. tostring(res))
        else
            -- ngx.log(ngx.ERR, "Loaded")
            configuration = res
        end

        local success, err = pcall(configuration.load_configuration_from_file)
        if not success then
            ngx.log(ngx.ERR, "Failed to load configuration from file: " .. tostring(err))
        else
            ngx.log(ngx.ERR, "Configuration loaded from file successfully in init phase.")
        end
    }

    init_worker_by_lua_block {
        local worker_id = ngx.worker.id()
        if worker_id == 0 then
            local timer_ok, timer_err = ngx.timer.at(0, configuration.check_for_updates)
            if not timer_ok then
                ngx.log(ngx.ERR, "Failed to create initial timer: " .. timer_err)
            else
                ngx.log(ngx.ERR, "Timer started successfully by worker with ID: " .. worker_id)
            end
        else
            ngx.log(ngx.ERR, "Worker with ID: " .. worker_id .. " is not starting the timer.")
        end
    }

    include       mime.types;
    default_type  application/octet-stream;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;
    server {
        # 测试用
        location /hello {
            default_type 'text/plain';
            content_by_lua 'ngx.say("hello, lua")';
        }
        # 配置接口
        # 通过http请求进行配置（发送json格式的post请求）
        location /configuration {
            client_max_body_size                    5m;
            client_body_buffer_size                 1m;
            proxy_buffering                         off;

            content_by_lua_block {
              configuration.call()
            }
        }
        # 查询共享内存中的path
        location /lua {
            default_type 'text/plain';
            # 读取请求中的 path 参数 并从共享 dict 中查询这个值，
            # 返回查询到的结果
            content_by_lua '
                ngx.log(ngx.ERR, "Entering rewrite_by_lua block")
                local path = ngx.req.get_uri_args()["path"]
                if path == nil then
                    ngx.say("path not found")
                    return
                end
                local data = ngx.shared.endpoints_data:get("/"..path)
                if not data then
                    ngx.say("unknown path")
                    return
                end
                ngx.say("paths: "..data)
            ';
        }
        
        # other path
        location / {
            set $load_ups "";

            # 动态设置当前 upstream, 未找到返回404
            rewrite_by_lua '
                local ups = configuration.getEndpoints()

                ngx.log(ngx.ERR, "upstream: ", ups)

                if ups ~= nil then
                    ngx.log(ngx.ERR,"got upstream: ", ups)
                    ngx.var.load_ups = ups
                    ngx.log(ngx.ERR, "Full proxy URL: " .. ngx.var.load_ups .. ngx.var.request_uri)

                    return
                end

                ngx.log(ngx.ERR, "No upstream found for path: " .. ngx.var.request_uri) -- 记录未找到的情况

                ngx.status = ngx.HTTP_NOT_FOUND
                ngx.exit(ngx.status)
            ';
            proxy_pass http://$load_ups;
            # add_header  X-Upstream  $upstream_addr always; # 添加 backend ip
        }
    }
}

lua脚本：

-- 引入变量
local io = io
local ngx = ngx
local table = table
-- 当前包的对象，类似 go 语言的定义结构体 让给这个结构体实现方法
local _M = {}
-- 与 Nginx 共享的空间 可读写
local Endpoints = ngx.shared.endpoints_data
local cjson = require "cjson"

if cjson then
    ngx.log(ngx.ERR, "cjson is loaded successfully.")
else
    ngx.log(ngx.ERR, "Failed to load cjson.")
end

local function split(inputstr, sep)
    sep = sep or "%s"
    local t = {}
    for str in string.gmatch(inputstr, "([^" .. sep .. "]+)") do
        table.insert(t, str)
    end
    return t
end

-- 读取请求 body 部分
local function fetch_request_body()
    ngx.req.read_body()
    local body = ngx.req.get_body_data()
  
    if not body then
      -- request body might've been written to tmp file if body > client_body_buffer_size
        local file_name = ngx.req.get_body_file()
        local file = io.open(file_name, "rb")
    
        if not file then
            return nil
        end
    
        body = file:read("*all")
        file:close()
    end
  
    return body
end

-- handle_backends .
local function handle_backends()
    if ngx.var.request_method == "GET" then
        ngx.status = ngx.HTTP_OK
        -- 返回查询的服务列表
        local path = ngx.req.get_uri_args()["path"]
        ngx.print(Endpoints:get(path))
        return
    end

    -- 读取请求 body
    local obj = fetch_request_body()
    if not obj then
        ngx.log(ngx.ERR, "dynamic-configuration: unable to read valid request body")
        ngx.status = ngx.HTTP_BAD_REQUEST
        return
    end

    -- 通过 第三方包 json 解析 body到 lua table
    ngx.log(ngx.ERR, "Received object: ", obj)
    local rule, err = cjson.decode(obj)
    if not rule then
        ngx.log(ngx.ERR, "could not parse backends data: ", err)
        return
    end

    ngx.log(ngx.ERR, "decoded rule", obj)

    -- 清空共享空间
    Endpoints:flush_all()
    -- 遍历并写入
    for _, new_rule in ipairs(rule.rules) do
        -- 更新
        -- 将数组合并
        local succ, err1, forcible = Endpoints:set(new_rule.path, table.concat(new_rule.upstreams, ","))
        ngx.log(ngx.ERR, "set result", succ, err1,forcible)
    end

    ngx.status = ngx.HTTP_CREATED
    ngx.say("ok")
end


-- call called by ngx

-- 从配置文件加载配置
function _M.load_configuration_from_file()
    local file = io.open("your/path/to/config.json", "r")  -- 指定配置文件路径
    if not file then
        ngx.log(ngx.ERR, "Failed to open configuration file.")
        return
    end

    local content = file:read("*all")
    file:close()

    local config, err = cjson.decode(content)
    if not config then
        ngx.log(ngx.ERR, "Failed to parse configuration: " .. err)
        return
    end

    ngx.log(ngx.ERR, "Parsed configuration: " .. cjson.encode(config))

    -- 清空共享空间
    Endpoints:flush_all()

    for _, rule in ipairs(config.rules) do
        ngx.log(ngx.ERR, "Loading rule - Path: " .. rule.path .. ", Upstreams: " .. table.concat(rule.upstreams, ","))
        -- 更新共享字典
        local succ, err1, forcible = Endpoints:set(rule.path, table.concat(rule.upstreams, ","))
        ngx.log(ngx.ERR, "Set result for path: " .. rule.path .. " - success: " .. tostring(succ) .. ", err: " .. tostring(err1))
    end

    ngx.log(ngx.ERR, "Configuration loaded from file successfully.")
end

-- 定时器回调函数
function _M.check_for_updates(premature)
    if premature then
        return
    end

    ngx.log(ngx.ERR, "check_for_updates is running")  -- 检查定时器任务是否运行

    _M.load_configuration_from_file() -- 调用函数加载配置

    -- 设置下一个定时器，设定间隔时间
    local ok, err = ngx.timer.at(0.1, _M.check_for_updates) -- 根据需要调整时间
    if not ok then
        ngx.log(ngx.ERR, "Failed to create timer: " .. err)
    end
end

function _M.call()
    -- 只处理 GET 和 POST
    if ngx.var.request_method ~= "POST" and ngx.var.request_method ~= "GET" then
        ngx.status = ngx.HTTP_BAD_REQUEST
        ngx.print("Only POST and GET requests are allowed!")
        return
      end
    -- 目前只处理后端服务的配置 所以判断路由
    if ngx.var.request_uri:find("/configuration/backends") then
        -- 调用内部方法
        handle_backends()
        return
    end
    -- 非法请求 返回 404
    ngx.status = ngx.HTTP_NOT_FOUND
    ngx.log(ngx.ERR, "Requested URI: ", ngx.var.request_uri)
    ngx.print("Not found!")
end



-- 轮询获取节点
function _M.getEndpoints() 
    local cache = ngx.shared.cache
    local path = ngx.var.request_uri
    local eps =  Endpoints:get(path)
    if not eps then
        return nil
    end

    local tab = split(eps,",")
    local index = cache:get(path)
    -- if index ==  nil or index > #tab then
    if index ==  nil or index > 2 then
        index = 1
    end
    -- 加一
    cache:set(path,index+1)
    return tab[index]
end

return _M 

转发路径文件

{
  "rules": [
    {
      "path": "/yourpath",
      "upstreams": ["backend1:8080", "backend2:8080"]
    }
  ]
}