qemu、rbd-nbd等客户端都是使用librbd进行ceph rbd卷的IO访问,如果要深入理解librbd,那么尝试自己写一个client来访问rbd卷(控制操作、IO操作),肯定是一个不错的途径。
写了个C的(异步IO模式),C++的可以参考:https://blog.csdn.net/JDPlus/article/details/76522298
测试ceph peering对客户端IO影响的场景下正好用到了异步IO工具,于是稍微改造了一下这个基于librbd的client工具:https://github.com/aspirer/study/blob/master/rbdclient.c
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 |
/* 注释就不详细写了,代码比较简单 */ #include <rados/librados.h> #include <rbd/librbd.h> #include <stdio.h> #include <stdlib.h> rados_t init_rados() { // we will use all of these below int ret = 0; rados_t rados = NULL; // 1. init rados object ret = rados_create(&rados, "admin"); // just use the client.admin keyring if (ret < 0) { // let's handle any error that might have come back printf("couldn't initialize rados! err %d\n", ret); return NULL; } else { printf("inited rados cluster object\n"); } return rados; } rados_ioctx_t init_ioctx(rados_t rados) { int ret = 0; rados_ioctx_t io_ctx = NULL; // 2. read ceph config file ret = rados_conf_read_file(rados, "/etc/ceph/ceph.conf"); if (ret < 0) { // This could fail if the config file is malformed, but it'd be hard. printf("failed to parse config file! err %d\n", ret); return NULL; } // 3. connect to ceph cluster ret = rados_connect(rados); if (ret < 0) { printf("couldn't connect to cluster! err %d\n", ret); return NULL; } else { printf("connected to the rados cluster\n"); } // 4. init io context for rbd pool const char *pool_name = "rbd"; ret = rados_ioctx_create(rados, pool_name, &io_ctx); if (ret < 0) { printf("couldn't setup ioctx! err %d\n", ret); rados_shutdown(rados); return NULL; } else { printf("created an ioctx for pool: rbd\n"); } return io_ctx; } rbd_image_t init_image(rados_ioctx_t io_ctx) { int ret = 0; // 5. open rbd image rbd_image_t image; const char *image_name = "sotest"; ret = rbd_open(io_ctx, image_name, &image, NULL); if (ret < 0) { printf("couldn't open rbd image! err %d\n", ret); return NULL; } else { printf("opened an rbd image: sotest\n"); } return image; } int get_rbd_size(rbd_image_t image) { int ret = 0; uint64_t size = 0; // 6. get rbd image size ret = rbd_get_size(image, &size); if (ret < 0) { printf("couldn't get image size! err %d\n", ret); return EXIT_FAILURE; } else { printf("The size of the image is: %dMB\n", size/1024/1024); } return size; } void rbd_finish_aiocb(rbd_completion_t c, void *arg) { // int ret = rbd_aio_wait_for_complete(c); int ret = rbd_aio_get_return_value(c); rbd_aio_release(c); // for aio read callback, the read data should be copied here to caller printf("aio callback: %d, %s\n", ret, (const char*)arg); } int aio_write(rbd_image_t image, const char *buff) { int off = 128; rbd_completion_t c; int ret = rbd_aio_create_completion((void *)buff, (rbd_callback_t) rbd_finish_aiocb, &c); if (ret < 0) { printf("create callback failed %s\n", ret); return ret; } int len = strlen(buff); ret = rbd_aio_write(image, off, len, buff, c); if (ret < 0) { printf("write to image failed %s\n", ret); return ret; } printf("write %s to image end\n", buff); return ret; } int aio_read(rbd_image_t image, char *buff) { int off = 128; int len = 10; rbd_completion_t c; int ret = rbd_aio_create_completion(buff, (rbd_callback_t) rbd_finish_aiocb, &c); if (ret < 0) { printf("create callback failed %s\n", ret); return ret; } memset(buff, 0, 128); ret = rbd_aio_read(image, off, len, buff, c); if (ret < 0) { printf("read from image failed %s\n", ret); return ret; } printf("read from image end\n"); return ret; } int main() { int ret; char buff[128] = {0}; int len; rados_t rados = init_rados(); if (!rados) { perror("init_rados"); return EXIT_FAILURE; } rados_ioctx_t io_ctx = init_ioctx(rados); if (!io_ctx) { perror("init_ioctx"); rados_shutdown(rados); return EXIT_FAILURE; } rbd_image_t image = init_image(io_ctx); if (!image) { perror("init_image"); rados_ioctx_destroy(io_ctx); rados_shutdown(rados); return EXIT_FAILURE; } int size = get_rbd_size(image); printf("image size: %d\n", size); sprintf(buff, "%s", "abcd123efg"); aio_write(image, buff); aio_read(image, buff); // 7. close image, io context and rados object ret = rbd_close(image); if (ret < 0) { printf("couldn't close rbd image! err %d\n", ret); return EXIT_FAILURE; } else { printf("closed rbd image: sotest\n"); } rados_ioctx_destroy(io_ctx); rados_shutdown(rados); return 0; } |
依赖包:apt-get install librbd-dev librados-dev
编译:gcc -g3 -O0 librbdtest.c -o librbdtest -lrados -lrbd
执行(需要先创建”rbd” pool和”sotest”卷):
1 2 3 4 5 6 7 8 9 10 11 12 |
$ ./librbdtest inited rados cluster object connected to the rados cluster created an ioctx for pool: rbd opened an rbd image: sotest The size of the image is: 128MB image size: 134217728 write abcd123efg to image end read from image end aio callback: 0, aio callback: 10, abcd123efg closed rbd image: sotest |
官方example:https://github.com/ceph/ceph/blob/master/examples/librbd/hello_world.cc