membase之ns_server监控数据采集

本文介绍了一种基于Couchbase系统的监控与统计机制。该机制利用master节点上的定时器发送事件,通过system_stats_collector收集CPU、内存等信息,并通过stats_collector获取memcached状态数据,最终将数据存入mnesia进行持久化存储。此外,还详细介绍了系统如何实现节点状态转换及自动接管。

摘要生成于 C知道 ,由 DeepSeek-R1 满血版支持, 前往体验 >

过程,由master的tick定时器定期往所有worker发送定时事件,systemstat和membasestat订阅这些事件然后定时上报状态

该tick守候进程只在master节点上启动

参与的类有

 

采集执行模块:

system_stats_collector 采集CPU,内存,磁盘情况

stats_collector 通过二进制命令往membase采集bucket数据

stats_archiver把采集的数据存储到mnesia中

stats_reader 读取模块

他们除了stats_reader外,其他的都是gen_server

 

辅助模块:

ns_tick 定时器守候进程,只在master运行

mb_master 逻辑模块,根据情况决定当前节点是master,candidate或者worker

mb_master_sup master节点的监控模块,把master节点需要的服务启动,如ns_tick

 

ns_pubsub 订阅辅助模块,采集执行体中相关的gen_server模块都是通过它从而获得订阅通知功能。


代码细节分析

mb_master判断和启动是否成为master

Master节点启动
init([]) ->
    Self = self(),
    ns_pubsub:subscribe(
      ns_config_events,
      fun ({nodes_wanted, Nodes}, State) ->
              Self ! {peers, Nodes},
              State;
          (_, State) ->
              State
      end, empty),
    erlang:process_flag(trap_exit, true),
    {ok, _} = timer:send_interval(?HEARTBEAT_INTERVAL, send_heartbeat),
    case ns_node_disco:nodes_wanted() of
        [N] = P when N == node() ->
            ?log_info("I'm the only node, so I'm the master.", []),
            {ok, master, start_master(#state{last_heard=now(), peers=P})};
        Peers when is_list(Peers) ->
            case lists:member(node(), Peers) of
                false ->
                    %% We're a worker, but don't know who the master is yet
                    ?log_info("Starting as worker. Peers: ~p", [Peers]),
                    {ok, worker, #state{last_heard=now()}};
                true ->
                    %% We're a candidate
                    ?log_info("Starting as candidate. Peers: ~p", [Peers]),
                    {ok, candidate, #state{last_heard=now(), peers=Peers}}
            end
end.
自动接管:
handle_info(send_heartbeat, candidate, #state{peers=Peers} = StateData) ->
    send_heartbeat(Peers, candidate, StateData),
    case timer:now_diff(now(), StateData#state.last_heard) >= ?TIMEOUT of
        true ->
            %% Take over
            ?log_info("Haven't heard from a higher priority node or "
                      "a master, so I'm taking over.", []),
            {ok, Pid} = mb_master_sup:start_link(),
            {next_state, master, StateData#state{child=Pid, master=node()}};
        false ->
            {next_state, candidate, StateData}
    end;

ns_tick对所有的节点发送统计命令

系统运行一个tick服务(gen_server),它定义了一个定时器(timer:send_interval(Interval, tick)),在定时器触发时对所有的node发出tick事件
handle_info(tick, State) ->
    misc:verify_name(?MODULE), % MB-3180: make sure we're still registered
    Now = misc:time_to_epoch_ms_int(now()),
    lists:foreach(fun (Node) ->
                          gen_event:notify({?EVENT_MANAGER, Node}, {tick, Now})
                  end, [node() | nodes()]),
{noreply, State#state{time=Now}};

通用的no embed事件订阅
-module(ns_pubsub).
-behaviour(gen_event).
subscribe(Name) ->
    subscribe(Name, msg_fun(self()), ignored).
subscribe(Name, Fun, State) ->
    Ref = make_ref(),
    ok = gen_event:add_sup_handler(Name, {?MODULE, Ref},
                                   #state{func=Fun, func_state=State}),
Ref.
msg_fun(Pid) ->
    fun (Event, ignored) ->
            Pid ! Event,
            ignored
end.
handle_event(Event, State = #state{func=Fun, func_state=FS}) ->
    NewState = Fun(Event, FS),
    {ok, State#state{func_state=NewState}};


系统监控:依赖于portsigar(sigar system-level stats for erlang) /RabbitMQ直接调用/usr/bin/vm_stat,/usr/sbin/prtconf和读取‘文件’"/proc/meminfo“

init([]) ->
    Path = path_config:component_path(bin, "sigar_port"),
    Port =
        try open_port({spawn_executable, Path},
                      [stream, use_stdio, exit_status,
                       binary, eof, {arg0, lists:flatten(io_lib:format("portsigar for ~s", [node()]))}]) of
            X ->
                ns_pubsub:subscribe(ns_tick_event),
                X
        catch error:enoent ->
                ?log_warning("~s is missing. Will not collect system-level stats", [Path]),
                undefined
        end,
    {ok, #state{port = Port}}.
handle_info({tick, TS}, #state{port = Port, prev_sample = PrevSample}) ->
    case flush_ticks(0) of
        0 -> ok;
        N -> ?log_warning("lost ~p ticks", [N])
    end,
    port_command(Port, <<0:32/native>>),
    Binary = recv_data(Port, [], ?STATS_BLOCK_SIZE),
    {Stats0, NewPrevSample} = unpack_data(Binary, PrevSample),
    case Stats0 of
        undefined -> ok;
        _ ->
            Stats = lists:sort(Stats0),
            gen_event:notify(ns_stats_event,
                             {stats, "@system", #stat_entry{timestamp = TS,
                                                            values = lists:sort(Stats)}})
    end,
{noreply, #state{port = Port, prev_sample = NewPrevSample}};

协议解析
unpack_data(Bin, PrevSample) ->
    <<Version:32/native,
      StructSize:32/native,
      CPULocalMS:64/native,
      CPUIdleMS:64/native,
      SwapTotal:64/native,
      SwapUsed:64/native,
      _SwapPageIn:64/native,
      _SwapPageOut:64/native,
      MemTotal:64/native,
      MemUsed:64/native,
      MemActualUsed:64/native,
      MemActualFree:64/native>> = Bin,
    StructSize = erlang:size(Bin),
    Version = 0,
    RawStats = [{cpu_local_ms, CPULocalMS},
                {cpu_idle_ms, CPUIdleMS},
                {swap_total, SwapTotal},
                {swap_used, SwapUsed},
                %% {swap_page_in, SwapPageIn},
                %% {swap_page_out, SwapPageOut},
                {mem_total, MemTotal},
                {mem_used, MemUsed},
                {mem_actual_used, MemActualUsed},
                {mem_actual_free, MemActualFree}],
    NowSamples = case PrevSample of
                     undefined -> undefined;
                     _ -> {_, OldCPULocal} = lists:keyfind(cpu_local_ms, 1, PrevSample),
                          {_, OldCPUIdle} = lists:keyfind(cpu_idle_ms, 1, PrevSample),
                          LocalDiff = CPULocalMS - OldCPULocal,
                          IdleDiff = CPUIdleMS - OldCPUIdle,
                          RV1 = lists:keyreplace(cpu_local_ms, 1, RawStats, {cpu_local_ms, LocalDiff}),
                          RV2 = lists:keyreplace(cpu_idle_ms, 1, RV1, {cpu_idle_ms, IdleDiff}),
                          [{mem_free, MemTotal - MemUsed},
                           {cpu_utilization_rate, try 100 * (LocalDiff - IdleDiff) / LocalDiff
                                                  catch error:badarith -> 0 end}
                           | RV2]
                 end,
{NowSamples, RawStats}.

stats_collector

连接本地memcached采集memcached状态数据。


持久化,stats_archived

do_handle_info({stats, Bucket, Sample}, State = #state{bucket=Bucket}) ->
    Tab = table(Bucket, minute),
    {atomic, ok} = mnesia:transaction(fun () ->
                                              mnesia:write(Tab, Sample, write)
                                      end, ?RETRIES),
    gen_event:notify(ns_stats_event, {sample_archived, Bucket, Sample}),
    {noreply, State};

stats_reader读取数据

resample(Bucket, Period, Step, N) ->
    Seconds = N * Step,
    Tab = stats_archiver:table(Bucket, Period),
    case mnesia:dirty_last(Tab) of
        '$end_of_table' ->
            {ok, []};
        Key ->
            Oldest = Key - Seconds * 1000 + 500,
            Handle = qlc:q([Sample || #stat_entry{timestamp=TS} = Sample
                                          <- mnesia:table(Tab), TS > Oldest]),
            F = fun (#stat_entry{timestamp = T} = Sample,
                     {T1, Acc, Chunk}) ->
                        case misc:trunc_ts(T, Step) of
                            T1 ->
                                {T1, Acc, [Sample|Chunk]};
                            T2 when T1 == undefined ->
                                {T2, Acc, [Sample]};
                            T2 ->
                                {T2, [avg(T1, Chunk)|Acc], [Sample]}
                        end
                end,
            case mnesia:activity(async_dirty, fun qlc:fold/3,
                                 [F, {undefined, [], []},
                                  Handle]) of
                {error, _, _} = Error ->
                    Error;
                {undefined, [], []} ->
                    {ok, []};
                {T, Acc, LastChunk} ->
                    {ok, lists:reverse([avg(T, LastChunk)|Acc])}
            end
    end.



Membase 是 NoSQL 家族的一个新的重量级的成员。 Membase是开源项目,源代码采用了Apache2.0的使用许可。该项目托管在GitHub.Source tarballs上,目前可以 下载beta版本的Linux二进制包。该产品主要是由North Scale的memcached核心团队成员开发完成,其中还包括Zynga和NHN这两个主要贡献者的工程师,这两个组织都是很大的在线游戏和社区网络空间的供应商。 Membase容易安装、操作,可以从单节点方便的扩展到集群,而且为memcached(有线协议的兼容性)实现了即插即用功能,在应用方面为开 发者和经营者提供了一个比较低的门槛。做为缓存解决方案,Memcached已经在不同类型的领域(特别是大容量的Web应用)有了广泛的使用,其中 Memcached的部分基础代码被直接应用到了Membase服务器的前端。 通过兼容多种编程语言和框架,Membase具备了很好的复用性。在安装和配置方面,Membase提供了有效的图形化界面和编程接口,包括可配置 的告警信息。 Membase的目标是提供对外的线性扩展能力,包括为了增加集群容量,可以针对统一的节点进行复制。 另外,对存储的数据进行再分配仍然是必要的。 这方面的一个有趣的特性是NoSQL解决方案所承诺的可预测的性能,类准确性的延迟和吞吐量。通过如下方式可以获得上面提到的特性: •自动将在线数据迁移到低延迟的存储介质的技术(内存,固态硬盘,磁盘) •可选的写操作一一异步,同步(基于复制,持久化) •反向通道再平衡[未来考虑支持] •多线程低锁争用 •尽可能使用异步处理 •自动实现重复数据删除 •动态再平衡现有集群 •通过把数据复制到多个集群单元和支持快速失败转移来提供系统的高可用性。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值