librbd库的使用




qemu、rbd-nbd等客户端都是使用librbd进行ceph rbd卷的IO访问,如果要深入理解librbd,那么尝试自己写一个client来访问rbd卷(控制操作、IO操作),肯定是一个不错的途径。

写了个C的(异步IO模式),C++的可以参考:https://blog.csdn.net/JDPlus/article/details/76522298

/* 注释就不详细写了,代码比较简单 */
#include <rados/librados.h>
#include <rbd/librbd.h>
#include <stdio.h>
#include <stdlib.h>


rados_t init_rados() {
    // we will use all of these below
    int ret = 0;
    rados_t rados = NULL;

    // 1. init rados object
    ret = rados_create(&rados, "admin"); // just use the client.admin keyring
    if (ret < 0) { // let's handle any error that might have come back
        printf("couldn't initialize rados! err %d\n", ret);
        return NULL;
    } else {
        printf("inited rados cluster object\n");
    }
    return rados;
}

rados_ioctx_t init_ioctx(rados_t rados) {
    int ret = 0;
    rados_ioctx_t io_ctx = NULL;

    // 2. read ceph config file
    ret = rados_conf_read_file(rados, "/etc/ceph/ceph.conf");
    if (ret < 0) {
        // This could fail if the config file is malformed, but it'd be hard.
        printf("failed to parse config file! err %d\n", ret);
        return NULL;
    }

    // 3. connect to ceph cluster
    ret = rados_connect(rados);
    if (ret < 0) {
        printf("couldn't connect to cluster! err %d\n", ret);
        return NULL;
    } else {
        printf("connected to the rados cluster\n");
    }

    // 4. init io context for rbd pool
    const char *pool_name = "rbd";
    ret = rados_ioctx_create(rados, pool_name, &io_ctx);
    if (ret < 0) {
        printf("couldn't setup ioctx! err %d\n", ret);
        rados_shutdown(rados);
        return NULL;
    } else {
        printf("created an ioctx for pool: rbd\n");
    }

    return io_ctx;
}

rbd_image_t init_image(rados_ioctx_t io_ctx) {
    int ret = 0;

    // 5. open rbd image
    rbd_image_t image;
    const char *image_name = "sotest";
    ret = rbd_open(io_ctx, image_name, &image, NULL);
    if (ret < 0) {
        printf("couldn't open rbd image! err %d\n", ret);
        return NULL;
    } else {
        printf("opened an rbd image: sotest\n");
    }

    return image;
}

int get_rbd_size(rbd_image_t image) {
    int ret = 0;
    uint64_t size = 0;

    // 6. get rbd image size
    ret = rbd_get_size(image, &size);
    if (ret < 0) {
        printf("couldn't get image size! err %d\n", ret);
        return EXIT_FAILURE;
    } else {
        printf("The size of the image is: %dMB\n", size/1024/1024);
    }

    return size;
}

void rbd_finish_aiocb(rbd_completion_t c, void *arg)
{
    // int ret = rbd_aio_wait_for_complete(c);
    int ret = rbd_aio_get_return_value(c);
    rbd_aio_release(c);

    // for aio read callback, the read data should be copied here to caller
    printf("aio callback: %d, %s\n", ret, (const char*)arg);
}

int aio_write(rbd_image_t image, const char *buff) {
    int off = 128;
    rbd_completion_t c;
    int ret = rbd_aio_create_completion((void *)buff, (rbd_callback_t) rbd_finish_aiocb, &c);
    if (ret < 0) {
        printf("create callback failed %s\n", ret);
        return ret;
    }

    int len = strlen(buff);
    ret = rbd_aio_write(image, off, len, buff, c);
    if (ret < 0) {
        printf("write to image failed %s\n", ret);
        return ret;
    }
    printf("write %s to image end\n", buff);

    return ret;
}

int aio_read(rbd_image_t image, char *buff) {
    int off = 128;
    int len = 10;
    rbd_completion_t c;
    int ret = rbd_aio_create_completion(buff, (rbd_callback_t) rbd_finish_aiocb, &c);
    if (ret < 0) {
        printf("create callback failed %s\n", ret);
        return ret;
    }
    memset(buff, 0, 128);
    ret = rbd_aio_read(image, off, len, buff, c);
    if (ret < 0) {
        printf("read from image failed %s\n", ret);
        return ret;
    }
    printf("read from image end\n");

    return ret;
}

int main() {
	int ret;
	char buff[128] = {0};
	int len;

    rados_t rados = init_rados();
    if (!rados) {
        perror("init_rados");
        return EXIT_FAILURE;
    }
    rados_ioctx_t io_ctx = init_ioctx(rados);
    if (!io_ctx) {
        perror("init_ioctx");
        rados_shutdown(rados);
        return EXIT_FAILURE;
    }

    rbd_image_t image = init_image(io_ctx);
    if (!image) {
        perror("init_image");
        rados_ioctx_destroy(io_ctx);
        rados_shutdown(rados);
        return EXIT_FAILURE;
    }

    int size = get_rbd_size(image);
    printf("image size: %d\n", size);

    sprintf(buff, "%s", "abcd123efg");
    aio_write(image, buff);

    aio_read(image, buff);

    // 7. close image, io context and rados object
    ret = rbd_close(image);
    if (ret < 0) {
        printf("couldn't close rbd image! err %d\n", ret);
        return EXIT_FAILURE;
    } else {
        printf("closed rbd image: sotest\n");
    }
    rados_ioctx_destroy(io_ctx);
    rados_shutdown(rados);

    return 0;

}

 

编译:gcc -g3 -O0 librbdtest.c -o librbdtest -lrados -lrbd

执行(需要先创建”rbd” pool和”sotest”卷):

$ ./librbdtest 
inited rados cluster object
connected to the rados cluster
created an ioctx for pool: rbd
opened an rbd image: sotest
The size of the image is: 128MB
image size: 134217728
write abcd123efg to image end
read  from image end
aio callback: 0, 
aio callback: 10, abcd123efg
closed rbd image: sotest

 

官方example:https://github.com/ceph/ceph/blob/master/examples/librbd/hello_world.cc