librbd 使用学习

qemu、rbd-nbd等客户端都是使用librbd进行ceph rbd卷的IO访问，如果要深入理解librbd，那么自己写一个client来访问rbd卷（控制操作、IO操作），肯定是个不错的学习方法。

 

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <rados/librados.h>
#include <rbd/librbd.h>

rbd_image_t init_image(rados_ioctx_t io_ctx) {
    int ret = 0;

    // 5. open rbd image
    rbd_image_t image;
    const char *image_name = "ee0c15655d44599a82a6bc0848b4aaf8";
    ret = rbd_open(io_ctx, image_name, &image, NULL);
    if (ret < 0) {
        printf("couldn't open rbd image! err %d\n", ret);
        return NULL;
    } else {
        printf("opened an rbd image: sotest\n");
    }

    return image;
}

int get_rbd_size(rbd_image_t image) {
    int ret = 0;
    uint64_t tsize = 0, size = 0;

    // 6. get rbd image size
    ret = rbd_get_size(image, &size);
    if (ret < 0) {
        printf("couldn't get image size! err %d\n", ret);
        return EXIT_FAILURE;
    } else {
        printf("The size of the image is: %dMB\n", size/1024/1024);
    }

    tsize = size;
    return tsize;
}

void rbd_finish_aiocb(rbd_completion_t c, void *arg)
{
    // int ret = rbd_aio_wait_for_complete(c);
    int ret = rbd_aio_get_return_value(c);
    rbd_aio_release(c);

    // for aio read callback, the read data should be copied here to caller
    printf("aio callback: %d, %s\n", ret, (const char*)arg);
}

ssize_t io_write(rbd_image_t image, const char *buff) {
    int off = 128;
    int len = strlen(buff);
    ssize_t ret;
    ret = rbd_write(rbd_image_t image, uint64_t ofs, size_t len,
                      const char *buf);

    if (ret < 0) {
        printf("write to image failed %s\n", ret);
        return ret;
    }
    printf("write %s to image end\n", buff);

    return ret;
}

int aio_write(rbd_image_t image, const char *buff) {
    int off = 128;
    rbd_completion_t c;
    int ret = rbd_aio_create_completion((void *)buff, (rbd_callback_t) rbd_finish_aiocb, &c);
    if (ret < 0) {
        printf("create callback failed %s\n", ret);
        return ret;
    }

    int len = strlen(buff);
    ret = rbd_aio_write(image, off, len, buff, c);
    if (ret < 0) {
        printf("write to image failed %s\n", ret);
        return ret;
    }
    printf("write %s to image end\n", buff);

    return ret;
}

ssize_t io_read(rbd_image_t image, char *buff) {
    int off = 128;
    int len = 10;
    ssize_t ret;
    ret = rbd_read(image, off, len, buff);
    if (ret < 0) {
        printf("read from image failed %s\n", ret);
        return ret;
    }
    printf("read from image end\n");

    return ret;
}

int aio_read(rbd_image_t image, char *buff) {
    int off = 128;
    int len = 10;
    rbd_completion_t c;
    int ret = rbd_aio_create_completion(buff, (rbd_callback_t) rbd_finish_aiocb, &c);
    if (ret < 0) {
        printf("create callback failed %s\n", ret);
        return ret;
    }
    memset(buff, 0, 128);
    ret = rbd_aio_read(image, off, len, buff, c);
    if (ret < 0) {
        printf("read from image failed %s\n", ret);
        return ret;
    }
    printf("read from image end\n");

    return ret;
}


int main (int argc, const char **argv)
{
        int ret;
        char buff[128] = {0};
        uint64_t size = 0;

        /* Declare the cluster handle and required arguments. */
        rados_t cluster;
        char cluster_name[] = "ceph";
        char user_name[] = "client.zstack";
        uint64_t flags = 0;

        /* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */
        int err;
        err = rados_create2(&cluster, cluster_name, user_name, flags);

        if (err < 0) {
                fprintf(stderr, "%s: Couldn't create the cluster handle! %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nCreated a cluster handle.\n");
        }


        /* Read a Ceph configuration file to configure the cluster handle. */
        err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
        if (err < 0) {
                fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nRead the config file.\n");
        }

        /* Read command line arguments */
        err = rados_conf_parse_argv(cluster, argc, argv);
        if (err < 0) {
                fprintf(stderr, "%s: cannot parse command line arguments: %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nRead the command line arguments.\n");
        }

        /* Connect to the cluster */
        err = rados_connect(cluster);
        if (err < 0) {
                fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nConnected to the cluster.\n");
        }

        rados_ioctx_t io_ctx;
        char *poolname = "data";

        err = rados_ioctx_create(cluster, poolname, &io_ctx);
        if (err < 0) {
                fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
                rados_shutdown(cluster);
                exit(EXIT_FAILURE);
        } else {
                printf("\nCreated I/O context.\n");
        }


    rbd_image_t image = init_image(io_ctx);
    if (!image) {
        perror("init_image");
        rados_ioctx_destroy(io_ctx);
        rados_shutdown(cluster);
        return EXIT_FAILURE;
    }

    size = get_rbd_size(image);
    printf("image size: %d\n", size);

    sprintf(buff, "%s", "abcd123efg");
    //aio_write(image, buff);

    aio_read(image, buff);

    // 7. close image, io context and rados object
    ret = rbd_close(image);
    if (ret < 0) {
        printf("couldn't close rbd image! err %d\n", ret);
        return EXIT_FAILURE;
    } else {
        printf("closed rbd image: sotest\n");
    }
    rados_ioctx_destroy(io_ctx);
    rados_shutdown(cluster);

    return 0;

}

编译： gcc -g test.c -o rbdtest -lrados -lrbd

[root@10-96-75-15 tmp]# ./ceph-client

Created a cluster handle.

Read the config file.

Read the command line arguments.

Connected to the cluster.

Created I/O context.
opened an rbd image: sotest
The size of the image is: 20480MB
image size: 0
read from image end
aio callback: 10,
closed rbd image: sotest

 

参考：https://docs.ceph.com/docs/master/rados/api/librados-intro/

          http://aspirer.wang/?p=1291