客户端压测server端计算qps以及不同延迟时间下响应数量所占百分比

将时间戳转为时间显示 date -d @978278400

percentile.h

https://github.com/chenshuo/muduo/blob/master/examples/sudoku/percentile.h

// this is not a standalone header file

class Percentile
{
 public:
  Percentile(std::vector<int>& latencies, int infly)
  {
    // latencies.size() 这1s的qps/rps,表示服务端处理多少个请求,infly表示有多少请求server没有处理完
    stat << "recv " << muduo::Fmt("%6zd", latencies.size()) << " in-fly " << infly;

    if (!latencies.empty())
    {
      // 将所有1s内server延迟时间排序
      std::sort(latencies.begin(), latencies.end());
      // 最小延迟时间
      int min = latencies.front();
      // 最大延迟时间
      int max = latencies.back();
      // 计算1s内延迟时间总和 
      int sum = std::accumulate(latencies.begin(), latencies.end(), 0);
      // 1s内响应延迟时间平均值
      int mean = sum / static_cast<int>(latencies.size());
      // 50%的请求延迟都小于median
      int median = getPercentile(latencies, 50);
      // 90%的延迟都小于p90
      int p90 = getPercentile(latencies, 90);
      // 90%的延迟都小于p99
      int p99 = getPercentile(latencies, 99);
      stat << " min " << min
           << " max " << max
           << " avg " << mean
           << " median " << median
           << " p90 " << p90
           << " p99 " << p99;
    }
  }

  const muduo::LogStream::Buffer& report() const
  {
    return stat.buffer();
  }

  void save(const std::vector<int>& latencies, muduo::StringArg name) const
  {
    if (latencies.empty())
      return;
    muduo::FileUtil::AppendFile f(name);
    f.append("# ", 2);
    f.append(stat.buffer().data(), stat.buffer().length());
    f.append("\n", 1);

    const int kInterval = 5; // 5 us per bucket
    int low = latencies.front() / kInterval * kInterval;
    int count = 0;
    int sum = 0;
    // 这1s的总共有多少个请求并且有响应
    const double total = static_cast<double>(latencies.size());
    char buf[64];
#ifndef NDEBUG
    for (size_t i = 0; i < latencies.size(); ++i)
    {
      int n = snprintf(buf, sizeof buf, "# %d\n", latencies[i]);
      f.append(buf, n);
    }
#endif
    // FIXME: improve this O(N) algorithm, maybe use lower_bound().
    // latencies此时已经排好序了
    for (size_t i = 0; i < latencies.size(); ++i)
    {
      if (latencies[i] < low + kInterval)
        ++count;
      else
      {
        sum += count;
        // count = [low, low+kInterval)这段时间有多少个请求,
        // 延迟时间小于low+kInterval的请求数量占总请求数量的百分比
        int n = snprintf(buf, sizeof buf, "%4d %5d %5.2f\n", low, count, 100 * sum / total);
        f.append(buf, n);
        low = latencies[i] / kInterval * kInterval;
        assert(latencies[i] < low + kInterval);
        count = 1;
      }
    }
    sum += count;
    assert(sum == total);
    int n = snprintf(buf, sizeof buf, "%4d %5d %5.1f\n", low, count, 100 * sum / total);
    f.append(buf, n);
  }

 private:

  static int getPercentile(const std::vector<int>& latencies, int percent)
  {
    // The Nearest Rank method
    assert(!latencies.empty());
    size_t idx = 0;
    if (percent > 0)
    {
      idx = (latencies.size() * percent + 99) / 100 - 1;
      assert(idx < latencies.size());
    }
    return latencies[idx];
  }

  muduo::LogStream stat;
};

loadtest.cc

https://github.com/chenshuo/muduo/blob/master/examples/sudoku/loadtest.cc

#include "examples/sudoku/sudoku.h"

#include "muduo/base/Logging.h"
#include "muduo/base/FileUtil.h"
#include "muduo/net/EventLoop.h"
#include "muduo/net/TcpClient.h"

#include <fstream>
#include <numeric>
#include <unordered_map>

#include "examples/sudoku/percentile.h"

#include <stdio.h>

using namespace muduo;
using namespace muduo::net;

typedef std::vector<string> Input;
typedef std::shared_ptr<const Input> InputPtr;

InputPtr readInput(std::istream& in)
{
  std::shared_ptr<Input> input(new Input);
  std::string line;
  while (getline(in, line))
  {
    if (line.size() == implicit_cast<size_t>(kCells))
    {
      input->push_back(line.c_str());
    }
  }
  return input;
}

class SudokuClient : noncopyable
{
 public:
  SudokuClient(EventLoop* loop,
               const InetAddress& serverAddr,
               const InputPtr& input,
               const string& name,
               bool nodelay)
    : name_(name),
      tcpNoDelay_(nodelay),
      client_(loop, serverAddr, name_),
      input_(input),
      count_(0)
  {
    client_.setConnectionCallback(
        std::bind(&SudokuClient::onConnection, this, _1));
    client_.setMessageCallback(
        std::bind(&SudokuClient::onMessage, this, _1, _2, _3));
  }

  void connect()
  {
    client_.connect();
  }

  void send(int n)
  {
    assert(n > 0);
    if (!conn_)
      return;

    Timestamp now(Timestamp::now());
    for (int i = 0; i < n; ++i)
    {
      char buf[256];
      // 计算需要文件的第几行为发送的请求参数
      const string& req = (*input_)[count_ % input_->size()];
      int len = snprintf(buf, sizeof buf, "%s-%08d:%s\r\n",
                         name_.c_str(), count_, req.c_str());
      requests_.append(buf, len);
      // 保留已经发送的请求数量,key为第几个发送的请求,value为发送的请求时间
      sendTime_[count_] = now;
      // 请求数量增加1
      ++count_;
    }

    conn_->send(&requests_);
  }

  void report(std::vector<int>* latency, int* infly)
  {
    // 这1s内所有请求的延迟
    latency->insert(latency->end(), latencies_.begin(), latencies_.end());
    // 清空上1s的延迟
    latencies_.clear();
    // 当前有多少请求没有收到
    *infly += static_cast<int>(sendTime_.size());
  }

 private:
  void onConnection(const TcpConnectionPtr& conn)
  {
    if (conn->connected())
    {
      LOG_INFO << name_ << " connected";
      if (tcpNoDelay_)
        conn->setTcpNoDelay(true);
      conn_ = conn;
    }
    else
    {
      LOG_INFO << name_ << " disconnected";
      conn_.reset();
      // FIXME: exit
    }
  }

  void onMessage(const TcpConnectionPtr& conn, Buffer* buf, Timestamp recvTime)
  {
    size_t len = buf->readableBytes();
    while (len >= kCells + 2)
    {
      const char* crlf = buf->findCRLF();
      if (crlf)
      {
        string response(buf->peek(), crlf);
        buf->retrieveUntil(crlf + 2);
        len = buf->readableBytes();
        if (!verify(response, recvTime))
        {
          LOG_ERROR << "Bad response:" << response;
          conn->shutdown();
          break;
        }
      }
      else if (len > 100) // id + ":" + kCells + "\r\n"
      {
        LOG_ERROR << "Line is too long!";
        conn->shutdown();
        break;
      }
      else
      {
        break;
      }
    }
  }

  bool verify(const string& response, Timestamp recvTime)
  {
    size_t colon = response.find(':');
    if (colon != string::npos)
    {
      size_t dash = response.find('-');
      if (dash != string::npos && dash < colon)
      {
        int id = atoi(response.c_str()+dash+1);
        // id 为发送的第几个请求,value为发送的请求时间
        std::unordered_map<int, Timestamp>::iterator sendTime = sendTime_.find(id);
        if (sendTime != sendTime_.end())
        {
          // id的请求响应延迟为latency_us
          int64_t latency_us = recvTime.microSecondsSinceEpoch() - sendTime->second.microSecondsSinceEpoch();
          // 将相应延迟加入latencies_
          latencies_.push_back(static_cast<int>(latency_us));
          // id为下表的请求已经收到了,从发送的请求中删除
          sendTime_.erase(sendTime);
        }
        else
        {
          LOG_ERROR << "Unknown id " << id << " of " << name_;
        }
      }
    }
    // FIXME
    return true;
  }

  const string name_;
  const bool tcpNoDelay_;
  TcpClient client_;
  TcpConnectionPtr conn_;
  Buffer requests_;
  const InputPtr input_;
  int count_;
  std::unordered_map<int, Timestamp> sendTime_;
  std::vector<int> latencies_;
};

class SudokuLoadtest : noncopyable
{
 public:
  SudokuLoadtest()
    : count_(0),
      ticks_(0),
      sofar_(0)
  {
  }

  void runClient(const InputPtr& input, const InetAddress& serverAddr, int rps, int conn, bool nodelay)
  {
    EventLoop loop;

    // 创建多少的连接到client端
    for (int i = 0; i < conn; ++i)
    {
      Fmt f("c%04d", i+1);
      string name(f.data(), f.length());
      clients_.emplace_back(new SudokuClient(&loop, serverAddr, input, name, nodelay));
      clients_.back()->connect();
    }

    // 0.01秒向server发送一次请求 
    loop.runEvery(1.0 / kHz, std::bind(&SudokuLoadtest::tick, this, rps));
    // client端看见每1s,server响应延迟latency,还有多少请求没有收到
    loop.runEvery(1.0, std::bind(&SudokuLoadtest::tock, this));
    loop.loop();
  }

 private:
  void tick(int rps)
  {
    // 执行发送请求的次数++
    ++ticks_;
    // 计算本次每个client应该发送的请求数量
    int64_t reqs = rps * ticks_ / kHz - sofar_;
    // 已经发送的总请求数量
    sofar_ += reqs;

    if (reqs > 0)
    {
      for (const auto& client : clients_)
      {
        client->send(static_cast<int>(reqs));
      }
    }
  }

  void tock()
  {
    std::vector<int> latencies;
    int infly = 0;
    for (const auto& client : clients_)
    {
      client->report(&latencies, &infly);
    }

    Percentile p(latencies, infly);
    // 计算这1s响应延迟的平均时间,最大延迟,最小延迟,50%的请求都在多少延迟之内
    LOG_INFO << p.report();
    char buf[64];
    snprintf(buf, sizeof buf, "r%04d", count_);
    p.save(latencies, buf);
    ++count_;
  }

  std::vector<std::unique_ptr<SudokuClient>> clients_;
  int count_;
  // 执行多少发送请求
  int64_t ticks_;
  // 已经发送的请求数量
  int64_t sofar_;
  static const int kHz = 100;
};

int main(int argc, char* argv[])
{
  int conn = 1;
  int rps = 100;
  bool nodelay = false;
  InetAddress serverAddr("127.0.0.1", 9981);
  switch (argc)
  {
    case 6:
      nodelay = string(argv[5]) == "-n";
      // FALL THROUGH
    case 5:
      conn = atoi(argv[4]);
      // FALL THROUGH
    case 4:
      rps = atoi(argv[3]);
      // FALL THROUGH
    case 3:
      serverAddr = InetAddress(argv[2], 9981);
      // FALL THROUGH
    case 2:
      break;
    default:
      printf("Usage: %s input server_ip [requests_per_second] [connections] [-n]\n", argv[0]);
      return 0;
  }

  std::ifstream in(argv[1]);
  if (in)
  {
    // vector<string> input; 每个index为文件的一行
    InputPtr input(readInput(in));
    printf("%zd requests from %s\n", input->size(), argv[1]);
    SudokuLoadtest test;
    test.runClient(input, serverAddr, rps, conn, nodelay);
  }
  else
  {
    printf("Cannot open %s\n", argv[1]);
  }
}