ceph存储底层代码走读

最新推荐文章于 2025-06-02 09:16:24 发布

原创最新推荐文章于 2025-06-02 09:16:24 发布 · 1.3k 阅读

2 ·

CC 4.0 BY-SA版权

文章标签：

#C++ #ceph #分布式

ceph 专栏收录该内容

23 篇文章

订阅专栏

上层ceph的机制，我这里就不详细解释了，当经过crush map计算后，落到对应的osd时，那么osd到底是如何将bufflist中的文件存储的呢？一言不合就上代码:)

在FileStore.cc中会有下面的代码，根据操作类型进行相关的逻辑：

//写操作

    case Transaction::OP_WRITE:
      {
        coll_t cid = i.get_cid(op->cid);
        ghobject_t oid = i.get_oid(op->oid);
   _kludge_temp_object_collection(cid, oid);
        uint64_t off = op->off;
        uint64_t len = op->len;
        uint32_t fadvise_flags = i.get_fadvise_flags();
        bufferlist bl;
        i.decode_bl(bl);
        tracepoint(objectstore, write_enter, osr_name, off, len);
        if (_check_replay_guard(cid, oid, spos) > 0)
          r = _write(cid, oid, off, len, bl, fadvise_flags);   ------------------------ 进入FileStore的_write函数
        tracepoint(objectstore, write_exit, r);
      }
      break;

int FileStore::_write(const coll_t& cid, const ghobject_t& oid,
                     uint64_t offset, size_t len,
                     const bufferlist& bl, uint32_t fadvise_flags)
{
dout(15) << "write " << cid << "/" << oid << " " << offset << "~" << len << dendl;
int r;

FDRef fd;
r = lfn_open(cid, oid, true, &fd); -------打开文件
if (r < 0) {
    dout(0) << "write couldn't open " << cid << "/"
        << oid << ": "
        << cpp_strerror(r) << dendl;
    goto out;
}

// write
r = bl.write_fd(**fd, offset); ---------写入文件，此处就调用了bufferlist的write_fd方法
if (r == 0)
    r = bl.length();

if (r >= 0 && m_filestore_sloppy_crc) {
    int rc = backend->_crc_update_write(**fd, offset, len, bl);
    assert(rc >= 0);
}

if (replaying || m_disable_wbthrottle) {
    if (fadvise_flags & CEPH_OSD_OP_FLAG_FADVISE_DONTNEED) {
        posix_fadvise(**fd, 0, 0, POSIX_FADV_DONTNEED);
    }
} else {
    wbthrottle.queue_wb(fd, oid, offset, len,
        fadvise_flags & CEPH_OSD_OP_FLAG_FADVISE_DONTNEED);
}

lfn_close(fd);

out:
dout(10) << "write " << cid << "/" << oid << " " << offset << "~" << len << " = " << r << dendl;
return r;
}

int buffer::list::write_fd(int fd, uint64_t offset) const
{
iovec iov[IOV_MAX];

std::list<ptr>::const_iterator p = _buffers.begin();
uint64_t left_pbrs = _buffers.size();
while (left_pbrs) {---------------------------------------循环读文件到一个iov

    ssize_t bytes = 0;
    unsigned iovlen = 0;
    uint64_t size = MIN(left_pbrs, IOV_MAX);
    left_pbrs -= size;
    while (size > 0) {
      iov[iovlen].iov_base = (void *)p->c_str();
      iov[iovlen].iov_len = p->length();
      iovlen++;
      bytes += p->length();
      ++p;
      size--;
    }

    int r = do_writev(fd, iov, offset, iovlen, bytes); ---------继续往下看
    if (r < 0)
      return r;
    offset += bytes;
}
return 0;
}

static int do_writev(int fd, struct iovec *vec, uint64_t offset, unsigned veclen, unsigned bytes)
{
ssize_t r = 0;
while (bytes > 0) {
#ifdef HAVE_PWRITEV
    r = ::pwritev(fd, vec, veclen, offset);
#else
    r = ::lseek64(fd, offset, SEEK_SET);-----------设置文件偏移量，接着之前的写。
    if (r != offset) {
      r = -errno;
      return r;
    }
    r = ::writev(fd, vec, veclen);---------------真正的写API，该函数可以自行百度查询，这里就不多BB了，下面付了写介绍。
#endif
    if (r < 0) {
      if (errno == EINTR)
        continue;
      return -errno;
    }

    bytes -= r;
    offset += r;
    if (bytes == 0) break;

    while (r > 0) {
      if (vec[0].iov_len <= (size_t)r) {
        // drain this whole item
        r -= vec[0].iov_len;
        ++vec;
        --veclen;
      } else {
        vec[0].iov_base = (char *)vec[0].iov_base + r;
        vec[0].iov_len -= r;
        break;
      }
    }
}
return 0;
}