ceph源码分析--admin socket

最新推荐文章于 2023-05-04 10:08:37 发布

原创最新推荐文章于 2023-05-04 10:08:37 发布 · 1.2k 阅读

2 ·

CC 4.0 BY-SA版权

文章标签：

#ceph #分布式系统

Ceph 专栏收录该内容

14 篇文章

订阅专栏

本文深入解析了 Ceph 存储系统中 Monitor 组件的工作原理及实现细节，包括其核心流程、服务线程启动机制及 Admin Socket 的初始化过程。

官方文档

http://docs.ceph.com/docs/master/rados/operations/monitoring/

具体用法

ceph daemon {daemon-name}
ceph daemon {path-to-socket-file}

ceph daemon osd.0 foo
ceph daemon /var/run/ceph/ceph-osd.0.asok foo

很多组件都可以通过命令，本文主要讲解Monitor部分
根据作者的理解这其实是一种UNIX Domain Socket IPC，具体的介绍可参阅《Unix环境高级编程》书籍

主要流程

这里写图片描述

common_init_finish(g_ceph_context);

void common_init_finish(CephContext *cct)

void common_init_finish(CephContext *cct)
{
  cct->init_crypto();
  ZTracer::ztrace_init();

  int flags = cct->get_init_flags();
  if (!(flags & CINIT_FLAG_NO_DAEMON_ACTIONS))
    //启动服务线程
    cct->start_service_thread();

  if ((flags & CINIT_FLAG_DEFER_DROP_PRIVILEGES) &&
      (cct->get_set_uid() || cct->get_set_gid())) {
    cct->get_admin_socket()->chown(cct->get_set_uid(), cct->get_set_gid());
  }

  md_config_t *conf = cct->_conf;

  if (!conf->admin_socket.empty() && !conf->admin_socket_mode.empty()) {
    int ret = 0;
    std::string err;

    ret = strict_strtol(conf->admin_socket_mode.c_str(), 8, &err);
    if (err.empty()) {
      if (!(ret & (~ACCESSPERMS))) {
        cct->get_admin_socket()->chmod(static_cast<mode_t>(ret));
      } else {
        lderr(cct) << "Invalid octal permissions string: "
            << conf->admin_socket_mode << dendl;
      }
    } else {
      lderr(cct) << "Invalid octal string: " << err << dendl;
    }
  }
}

start_service_thread()

void CephContext::start_service_thread()
{
  ceph_spin_lock(&_service_thread_lock);
  if (_service_thread) {
    ceph_spin_unlock(&_service_thread_lock);
    return;
  }
  _service_thread = new CephContextServiceThread(this);
  _service_thread->create("service");
  ceph_spin_unlock(&_service_thread_lock);

  // make logs flush on_exit()
  if (_conf->log_flush_on_exit)
    _log->set_flush_on_exit();

  // Trigger callbacks on any config observers that were waiting for
  // it to become safe to start threads.
  _conf->set_val("internal_safe_to_start_threads", "true");
  _conf->call_all_observers();

  // start admin socket
  if (_conf->admin_socket.length())
    //初始化
    _admin_socket->init(_conf->admin_socket);
}

AdminSocket::init(const std::string &path)

bool AdminSocket::init(const std::string &path)
{
  ldout(m_cct, 5) << "init " << path << dendl;

  /* Set up things for the new thread */
  std::string err;
  int pipe_rd = -1, pipe_wr = -1;
  //创建管道用于关闭
  err = create_shutdown_pipe(&pipe_rd, &pipe_wr);
  if (!err.empty()) {
    lderr(m_cct) << "AdminSocketConfigObs::init: error: " << err << dendl;
    return false;
  }
  int sock_fd;
  //绑定监听
  err = bind_and_listen(path, &sock_fd);
  if (!err.empty()) {
    lderr(m_cct) << "AdminSocketConfigObs::init: failed: " << err << dendl;
    close(pipe_rd);
    close(pipe_wr);
    return false;
  }

  /* Create new thread */
  m_sock_fd = sock_fd;
  m_shutdown_rd_fd = pipe_rd;
  m_shutdown_wr_fd = pipe_wr;
  m_path = path;

  m_version_hook = new VersionHook;
  register_command("0", "0", m_version_hook, "");
  register_command("version", "version", m_version_hook, "get ceph version");
  register_command("git_version", "git_version", m_version_hook, "get git sha1");
  m_help_hook = new HelpHook(this);
  register_command("help", "help", m_help_hook, "list available commands");
  m_getdescs_hook = new GetdescsHook(this);
  register_command("get_command_descriptions", "get_command_descriptions",
           m_getdescs_hook, "list available commands");
  //创建一个线程，运行entry()
  create("admin_socket");
  add_cleanup_file(m_path.c_str());
  return true;
}

AdminSocket::entry()

void* AdminSocket::entry()
{
  ldout(m_cct, 5) << "entry start" << dendl;
  while (true) {
    struct pollfd fds[2];
    memset(fds, 0, sizeof(fds));
    fds[0].fd = m_sock_fd;
    fds[0].events = POLLIN | POLLRDBAND;
    fds[1].fd = m_shutdown_rd_fd;
    fds[1].events = POLLIN | POLLRDBAND;
    //poll轮询
    int ret = poll(fds, 2, -1);
    if (ret < 0) {
      int err = errno;
      if (err == EINTR) {
    continue;
      }
      lderr(m_cct) << "AdminSocket: poll(2) error: '"
           << cpp_strerror(err) << dendl;
      return PFL_FAIL;
    }

    if (fds[0].revents & POLLIN) {
      // Send out some data
      //处理请求
      do_accept();
    }
    if (fds[1].revents & POLLIN) {
      // Parent wants us to shut down
      //父进程希望关闭
      return PFL_SUCCESS;
    }
  }
  ldout(m_cct, 5) << "entry exit" << dendl;
}

AdminSocket::do_accept()

while (1) {
    //读取命令
    int ret = safe_read(connection_fd, &cmd[pos], 1);
    if (ret <= 0) {
      if (ret < 0) {
        lderr(m_cct) << "AdminSocket: error reading request code: "
             << cpp_strerror(ret) << dendl;
      }
···
//寻找对应的hook
while (match.size()) {
    p = m_hooks.find(match);
    if (p != m_hooks.end())
      break;

    // drop right-most word
    size_t pos = match.rfind(' ');
    if (pos == std::string::npos) {
      match.clear();  // we fail
      break;
    } else {
      match.resize(pos);
    }
  }
···
    in_hook = true;
    auto match_hook = p->second;
    m_lock.Unlock();
    //调用子类的call函数
    bool success = match_hook->call(match, cmdmap, format, out);
    m_lock.Lock();
···
//返回结果
int ret = safe_write(connection_fd, &len, sizeof(len));
  ···
if (out.write_fd(connection_fd) >= 0)